drivers/dma/ppc4xx/adma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2006-2009 DENX Software Engineering.
   4  *
   5  * Author: Yuri Tikhonov <yur@emcraft.com>
   6  *
   7  * Further porting to arch/powerpc by
   8  *      Anatolij Gustschin <agust@denx.de>
   9  */
  10
  11 /*
  12  * This driver supports the asynchrounous DMA copy and RAID engines available
  13  * on the AMCC PPC440SPe Processors.
  14  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  15  * ADMA driver written by D.Williams.
  16  */
  17
  18 #include <linux/init.h>
  19 #include <linux/module.h>
  20 #include <linux/async_tx.h>
  21 #include <linux/delay.h>
  22 #include <linux/dma-mapping.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/slab.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/proc_fs.h>
  28 #include <linux/of.h>
  29 #include <linux/of_address.h>
  30 #include <linux/of_irq.h>
  31 #include <linux/of_platform.h>
  32 #include <asm/dcr.h>
  33 #include <asm/dcr-regs.h>
  34 #include "adma.h"
  35 #include "../dmaengine.h"
  36
  37 enum ppc_adma_init_code {
  38         PPC_ADMA_INIT_OK = 0,
  39         PPC_ADMA_INIT_MEMRES,
  40         PPC_ADMA_INIT_MEMREG,
  41         PPC_ADMA_INIT_ALLOC,
  42         PPC_ADMA_INIT_COHERENT,
  43         PPC_ADMA_INIT_CHANNEL,
  44         PPC_ADMA_INIT_IRQ1,
  45         PPC_ADMA_INIT_IRQ2,
  46         PPC_ADMA_INIT_REGISTER
  47 };
  48
  49 static char *ppc_adma_errors[] = {
  50         [PPC_ADMA_INIT_OK] = "ok",
  51         [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  52         [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  53         [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  54                                 "structure",
  55         [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  56                                    "hardware descriptors",
  57         [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  58         [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  59         [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  60         [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  61 };
  62
  63 static enum ppc_adma_init_code
  64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  65
  66 struct ppc_dma_chan_ref {
  67         struct dma_chan *chan;
  68         struct list_head node;
  69 };
  70
  71 /* The list of channels exported by ppc440spe ADMA */
  72 static struct list_head
  73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  74
  75 /* This flag is set when want to refetch the xor chain in the interrupt
  76  * handler
  77  */
  78 static u32 do_xor_refetch;
  79
  80 /* Pointer to DMA0, DMA1 CP/CS FIFO */
  81 static void *ppc440spe_dma_fifo_buf;
  82
  83 /* Pointers to last submitted to DMA0, DMA1 CDBs */
  84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
  85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
  86
  87 /* Pointer to last linked and submitted xor CB */
  88 static struct ppc440spe_adma_desc_slot *xor_last_linked;
  89 static struct ppc440spe_adma_desc_slot *xor_last_submit;
  90
  91 /* This array is used in data-check operations for storing a pattern */
  92 static char ppc440spe_qword[16];
  93
  94 static atomic_t ppc440spe_adma_err_irq_ref;
  95 static dcr_host_t ppc440spe_mq_dcr_host;
  96 static unsigned int ppc440spe_mq_dcr_len;
  97
  98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
  99  * the block size in transactions, then we do not allow to activate more than
 100  * only one RXOR transactions simultaneously. So use this var to store
 101  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
 102  * set) or not (PPC440SPE_RXOR_RUN is clear).
 103  */
 104 static unsigned long ppc440spe_rxor_state;
 105
 106 /* These are used in enable & check routines
 107  */
 108 static u32 ppc440spe_r6_enabled;
 109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
 110 static struct completion ppc440spe_r6_test_comp;
 111
 112 static int ppc440spe_adma_dma2rxor_prep_src(
 113                 struct ppc440spe_adma_desc_slot *desc,
 114                 struct ppc440spe_rxor *cursor, int index,
 115                 int src_cnt, u32 addr);
 116 static void ppc440spe_adma_dma2rxor_set_src(
 117                 struct ppc440spe_adma_desc_slot *desc,
 118                 int index, dma_addr_t addr);
 119 static void ppc440spe_adma_dma2rxor_set_mult(
 120                 struct ppc440spe_adma_desc_slot *desc,
 121                 int index, u8 mult);
 122
 123 #ifdef ADMA_LL_DEBUG
 124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
 125 #else
 126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
 127 #endif
 128
 129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
 130 {
 131         struct dma_cdb *cdb;
 132         struct xor_cb *cb;
 133         int i;
 134
 135         switch (chan->device->id) {
 136         case 0:
 137         case 1:
 138                 cdb = block;
 139
 140                 pr_debug("CDB at %p [%d]:\n"
 141                         "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
 142                         "\t sg1u 0x%08x sg1l 0x%08x\n"
 143                         "\t sg2u 0x%08x sg2l 0x%08x\n"
 144                         "\t sg3u 0x%08x sg3l 0x%08x\n",
 145                         cdb, chan->device->id,
 146                         cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
 147                         le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
 148                         le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
 149                         le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
 150                 );
 151                 break;
 152         case 2:
 153                 cb = block;
 154
 155                 pr_debug("CB at %p [%d]:\n"
 156                         "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
 157                         "\t cbtah 0x%08x cbtal 0x%08x\n"
 158                         "\t cblah 0x%08x cblal 0x%08x\n",
 159                         cb, chan->device->id,
 160                         cb->cbc, cb->cbbc, cb->cbs,
 161                         cb->cbtah, cb->cbtal,
 162                         cb->cblah, cb->cblal);
 163                 for (i = 0; i < 16; i++) {
 164                         if (i && !cb->ops[i].h && !cb->ops[i].l)
 165                                 continue;
 166                         pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
 167                                 i, cb->ops[i].h, cb->ops[i].l);
 168                 }
 169                 break;
 170         }
 171 }
 172
 173 static void print_cb_list(struct ppc440spe_adma_chan *chan,
 174                           struct ppc440spe_adma_desc_slot *iter)
 175 {
 176         for (; iter; iter = iter->hw_next)
 177                 print_cb(chan, iter->hw_desc);
 178 }
 179
 180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
 181                              unsigned int src_cnt)
 182 {
 183         int i;
 184
 185         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 186         for (i = 0; i < src_cnt; i++)
 187                 pr_debug("\t0x%016llx ", src[i]);
 188         pr_debug("dst:\n\t0x%016llx\n", dst);
 189 }
 190
 191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
 192                             unsigned int src_cnt)
 193 {
 194         int i;
 195
 196         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 197         for (i = 0; i < src_cnt; i++)
 198                 pr_debug("\t0x%016llx ", src[i]);
 199         pr_debug("dst: ");
 200         for (i = 0; i < 2; i++)
 201                 pr_debug("\t0x%016llx ", dst[i]);
 202 }
 203
 204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
 205                                     unsigned int src_cnt,
 206                                     const unsigned char *scf)
 207 {
 208         int i;
 209
 210         pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
 211         if (scf) {
 212                 for (i = 0; i < src_cnt; i++)
 213                         pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
 214         } else {
 215                 for (i = 0; i < src_cnt; i++)
 216                         pr_debug("\t0x%016llx(no) ", src[i]);
 217         }
 218
 219         pr_debug("dst: ");
 220         for (i = 0; i < 2; i++)
 221                 pr_debug("\t0x%016llx ", src[src_cnt + i]);
 222 }
 223
 224 /******************************************************************************
 225  * Command (Descriptor) Blocks low-level routines
 226  ******************************************************************************/
 227 /**
 228  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
 229  * pseudo operation
 230  */
 231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
 232                                           struct ppc440spe_adma_chan *chan)
 233 {
 234         struct xor_cb *p;
 235
 236         switch (chan->device->id) {
 237         case PPC440SPE_XOR_ID:
 238                 p = desc->hw_desc;
 239                 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 240                 /* NOP with Command Block Complete Enable */
 241                 p->cbc = XOR_CBCR_CBCE_BIT;
 242                 break;
 243         case PPC440SPE_DMA0_ID:
 244         case PPC440SPE_DMA1_ID:
 245                 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 246                 /* NOP with interrupt */
 247                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 248                 break;
 249         default:
 250                 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
 251                                 __func__);
 252                 break;
 253         }
 254 }
 255
 256 /**
 257  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
 258  * pseudo operation
 259  */
 260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
 261 {
 262         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 263         desc->hw_next = NULL;
 264         desc->src_cnt = 0;
 265         desc->dst_cnt = 1;
 266 }
 267
 268 /**
 269  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
 270  */
 271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
 272                                          int src_cnt, unsigned long flags)
 273 {
 274         struct xor_cb *hw_desc = desc->hw_desc;
 275
 276         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 277         desc->hw_next = NULL;
 278         desc->src_cnt = src_cnt;
 279         desc->dst_cnt = 1;
 280
 281         hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
 282         if (flags & DMA_PREP_INTERRUPT)
 283                 /* Enable interrupt on completion */
 284                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 285 }
 286
 287 /**
 288  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
 289  * operation in DMA2 controller
 290  */
 291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
 292                 int dst_cnt, int src_cnt, unsigned long flags)
 293 {
 294         struct xor_cb *hw_desc = desc->hw_desc;
 295
 296         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 297         desc->hw_next = NULL;
 298         desc->src_cnt = src_cnt;
 299         desc->dst_cnt = dst_cnt;
 300         memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
 301         desc->descs_per_op = 0;
 302
 303         hw_desc->cbc = XOR_CBCR_TGT_BIT;
 304         if (flags & DMA_PREP_INTERRUPT)
 305                 /* Enable interrupt on completion */
 306                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 307 }
 308
 309 #define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
 310 #define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
 311 #define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
 312
 313 /**
 314  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
 315  * with DMA0/1
 316  */
 317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
 318                                 int dst_cnt, int src_cnt, unsigned long flags,
 319                                 unsigned long op)
 320 {
 321         struct dma_cdb *hw_desc;
 322         struct ppc440spe_adma_desc_slot *iter;
 323         u8 dopc;
 324
 325         /* Common initialization of a PQ descriptors chain */
 326         set_bits(op, &desc->flags);
 327         desc->src_cnt = src_cnt;
 328         desc->dst_cnt = dst_cnt;
 329
 330         /* WXOR MULTICAST if both P and Q are being computed
 331          * MV_SG1_SG2 if Q only
 332          */
 333         dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
 334                 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
 335
 336         list_for_each_entry(iter, &desc->group_list, chain_node) {
 337                 hw_desc = iter->hw_desc;
 338                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 339
 340                 if (likely(!list_is_last(&iter->chain_node,
 341                                 &desc->group_list))) {
 342                         /* set 'next' pointer */
 343                         iter->hw_next = list_entry(iter->chain_node.next,
 344                                 struct ppc440spe_adma_desc_slot, chain_node);
 345                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 346                 } else {
 347                         /* this is the last descriptor.
 348                          * this slot will be pasted from ADMA level
 349                          * each time it wants to configure parameters
 350                          * of the transaction (src, dst, ...)
 351                          */
 352                         iter->hw_next = NULL;
 353                         if (flags & DMA_PREP_INTERRUPT)
 354                                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
 355                         else
 356                                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 357                 }
 358         }
 359
 360         /* Set OPS depending on WXOR/RXOR type of operation */
 361         if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
 362                 /* This is a WXOR only chain:
 363                  * - first descriptors are for zeroing destinations
 364                  *   if PPC440SPE_ZERO_P/Q set;
 365                  * - descriptors remained are for GF-XOR operations.
 366                  */
 367                 iter = list_first_entry(&desc->group_list,
 368                                         struct ppc440spe_adma_desc_slot,
 369                                         chain_node);
 370
 371                 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
 372                         hw_desc = iter->hw_desc;
 373                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 374                         iter = list_first_entry(&iter->chain_node,
 375                                         struct ppc440spe_adma_desc_slot,
 376                                         chain_node);
 377                 }
 378
 379                 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
 380                         hw_desc = iter->hw_desc;
 381                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 382                         iter = list_first_entry(&iter->chain_node,
 383                                         struct ppc440spe_adma_desc_slot,
 384                                         chain_node);
 385                 }
 386
 387                 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 388                         hw_desc = iter->hw_desc;
 389                         hw_desc->opc = dopc;
 390                 }
 391         } else {
 392                 /* This is either RXOR-only or mixed RXOR/WXOR */
 393
 394                 /* The first 1 or 2 slots in chain are always RXOR,
 395                  * if need to calculate P & Q, then there are two
 396                  * RXOR slots; if only P or only Q, then there is one
 397                  */
 398                 iter = list_first_entry(&desc->group_list,
 399                                         struct ppc440spe_adma_desc_slot,
 400                                         chain_node);
 401                 hw_desc = iter->hw_desc;
 402                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 403
 404                 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
 405                         iter = list_first_entry(&iter->chain_node,
 406                                                 struct ppc440spe_adma_desc_slot,
 407                                                 chain_node);
 408                         hw_desc = iter->hw_desc;
 409                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 410                 }
 411
 412                 /* The remaining descs (if any) are WXORs */
 413                 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
 414                         iter = list_first_entry(&iter->chain_node,
 415                                                 struct ppc440spe_adma_desc_slot,
 416                                                 chain_node);
 417                         list_for_each_entry_from(iter, &desc->group_list,
 418                                                 chain_node) {
 419                                 hw_desc = iter->hw_desc;
 420                                 hw_desc->opc = dopc;
 421                         }
 422                 }
 423         }
 424 }
 425
 426 /**
 427  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
 428  * for PQ_ZERO_SUM operation
 429  */
 430 static void ppc440spe_desc_init_dma01pqzero_sum(
 431                                 struct ppc440spe_adma_desc_slot *desc,
 432                                 int dst_cnt, int src_cnt)
 433 {
 434         struct dma_cdb *hw_desc;
 435         struct ppc440spe_adma_desc_slot *iter;
 436         int i = 0;
 437         u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
 438                                    DMA_CDB_OPC_MV_SG1_SG2;
 439         /*
 440          * Initialize starting from 2nd or 3rd descriptor dependent
 441          * on dst_cnt. First one or two slots are for cloning P
 442          * and/or Q to chan->pdest and/or chan->qdest as we have
 443          * to preserve original P/Q.
 444          */
 445         iter = list_first_entry(&desc->group_list,
 446                                 struct ppc440spe_adma_desc_slot, chain_node);
 447         iter = list_entry(iter->chain_node.next,
 448                           struct ppc440spe_adma_desc_slot, chain_node);
 449
 450         if (dst_cnt > 1) {
 451                 iter = list_entry(iter->chain_node.next,
 452                                   struct ppc440spe_adma_desc_slot, chain_node);
 453         }
 454         /* initialize each source descriptor in chain */
 455         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 456                 hw_desc = iter->hw_desc;
 457                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 458                 iter->src_cnt = 0;
 459                 iter->dst_cnt = 0;
 460
 461                 /* This is a ZERO_SUM operation:
 462                  * - <src_cnt> descriptors starting from 2nd or 3rd
 463                  *   descriptor are for GF-XOR operations;
 464                  * - remaining <dst_cnt> descriptors are for checking the result
 465                  */
 466                 if (i++ < src_cnt)
 467                         /* MV_SG1_SG2 if only Q is being verified
 468                          * MULTICAST if both P and Q are being verified
 469                          */
 470                         hw_desc->opc = dopc;
 471                 else
 472                         /* DMA_CDB_OPC_DCHECK128 operation */
 473                         hw_desc->opc = DMA_CDB_OPC_DCHECK128;
 474
 475                 if (likely(!list_is_last(&iter->chain_node,
 476                                          &desc->group_list))) {
 477                         /* set 'next' pointer */
 478                         iter->hw_next = list_entry(iter->chain_node.next,
 479                                                 struct ppc440spe_adma_desc_slot,
 480                                                 chain_node);
 481                 } else {
 482                         /* this is the last descriptor.
 483                          * this slot will be pasted from ADMA level
 484                          * each time it wants to configure parameters
 485                          * of the transaction (src, dst, ...)
 486                          */
 487                         iter->hw_next = NULL;
 488                         /* always enable interrupt generation since we get
 489                          * the status of pqzero from the handler
 490                          */
 491                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
 492                 }
 493         }
 494         desc->src_cnt = src_cnt;
 495         desc->dst_cnt = dst_cnt;
 496 }
 497
 498 /**
 499  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
 500  */
 501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
 502                                         unsigned long flags)
 503 {
 504         struct dma_cdb *hw_desc = desc->hw_desc;
 505
 506         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 507         desc->hw_next = NULL;
 508         desc->src_cnt = 1;
 509         desc->dst_cnt = 1;
 510
 511         if (flags & DMA_PREP_INTERRUPT)
 512                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 513         else
 514                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 515
 516         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 517 }
 518
 519 /**
 520  * ppc440spe_desc_set_src_addr - set source address into the descriptor
 521  */
 522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
 523                                         struct ppc440spe_adma_chan *chan,
 524                                         int src_idx, dma_addr_t addrh,
 525                                         dma_addr_t addrl)
 526 {
 527         struct dma_cdb *dma_hw_desc;
 528         struct xor_cb *xor_hw_desc;
 529         phys_addr_t addr64, tmplow, tmphi;
 530
 531         switch (chan->device->id) {
 532         case PPC440SPE_DMA0_ID:
 533         case PPC440SPE_DMA1_ID:
 534                 if (!addrh) {
 535                         addr64 = addrl;
 536                         tmphi = (addr64 >> 32);
 537                         tmplow = (addr64 & 0xFFFFFFFF);
 538                 } else {
 539                         tmphi = addrh;
 540                         tmplow = addrl;
 541                 }
 542                 dma_hw_desc = desc->hw_desc;
 543                 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
 544                 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
 545                 break;
 546         case PPC440SPE_XOR_ID:
 547                 xor_hw_desc = desc->hw_desc;
 548                 xor_hw_desc->ops[src_idx].l = addrl;
 549                 xor_hw_desc->ops[src_idx].h |= addrh;
 550                 break;
 551         }
 552 }
 553
 554 /**
 555  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
 556  */
 557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
 558                         struct ppc440spe_adma_chan *chan, u32 mult_index,
 559                         int sg_index, unsigned char mult_value)
 560 {
 561         struct dma_cdb *dma_hw_desc;
 562         u32 *psgu;
 563
 564         switch (chan->device->id) {
 565         case PPC440SPE_DMA0_ID:
 566         case PPC440SPE_DMA1_ID:
 567                 dma_hw_desc = desc->hw_desc;
 568
 569                 switch (sg_index) {
 570                 /* for RXOR operations set multiplier
 571                  * into source cued address
 572                  */
 573                 case DMA_CDB_SG_SRC:
 574                         psgu = &dma_hw_desc->sg1u;
 575                         break;
 576                 /* for WXOR operations set multiplier
 577                  * into destination cued address(es)
 578                  */
 579                 case DMA_CDB_SG_DST1:
 580                         psgu = &dma_hw_desc->sg2u;
 581                         break;
 582                 case DMA_CDB_SG_DST2:
 583                         psgu = &dma_hw_desc->sg3u;
 584                         break;
 585                 default:
 586                         BUG();
 587                 }
 588
 589                 *psgu |= cpu_to_le32(mult_value << mult_index);
 590                 break;
 591         case PPC440SPE_XOR_ID:
 592                 break;
 593         default:
 594                 BUG();
 595         }
 596 }
 597
 598 /**
 599  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
 600  */
 601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 602                                 struct ppc440spe_adma_chan *chan,
 603                                 dma_addr_t addrh, dma_addr_t addrl,
 604                                 u32 dst_idx)
 605 {
 606         struct dma_cdb *dma_hw_desc;
 607         struct xor_cb *xor_hw_desc;
 608         phys_addr_t addr64, tmphi, tmplow;
 609         u32 *psgu, *psgl;
 610
 611         switch (chan->device->id) {
 612         case PPC440SPE_DMA0_ID:
 613         case PPC440SPE_DMA1_ID:
 614                 if (!addrh) {
 615                         addr64 = addrl;
 616                         tmphi = (addr64 >> 32);
 617                         tmplow = (addr64 & 0xFFFFFFFF);
 618                 } else {
 619                         tmphi = addrh;
 620                         tmplow = addrl;
 621                 }
 622                 dma_hw_desc = desc->hw_desc;
 623
 624                 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
 625                 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
 626
 627                 *psgl = cpu_to_le32((u32)tmplow);
 628                 *psgu |= cpu_to_le32((u32)tmphi);
 629                 break;
 630         case PPC440SPE_XOR_ID:
 631                 xor_hw_desc = desc->hw_desc;
 632                 xor_hw_desc->cbtal = addrl;
 633                 xor_hw_desc->cbtah |= addrh;
 634                 break;
 635         }
 636 }
 637
 638 /**
 639  * ppc440spe_desc_set_byte_count - set number of data bytes involved
 640  * into the operation
 641  */
 642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
 643                                 struct ppc440spe_adma_chan *chan,
 644                                 u32 byte_count)
 645 {
 646         struct dma_cdb *dma_hw_desc;
 647         struct xor_cb *xor_hw_desc;
 648
 649         switch (chan->device->id) {
 650         case PPC440SPE_DMA0_ID:
 651         case PPC440SPE_DMA1_ID:
 652                 dma_hw_desc = desc->hw_desc;
 653                 dma_hw_desc->cnt = cpu_to_le32(byte_count);
 654                 break;
 655         case PPC440SPE_XOR_ID:
 656                 xor_hw_desc = desc->hw_desc;
 657                 xor_hw_desc->cbbc = byte_count;
 658                 break;
 659         }
 660 }
 661
 662 /**
 663  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
 664  */
 665 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
 666 {
 667         /* assume that byte_count is aligned on the 512-boundary;
 668          * thus write it directly to the register (bits 23:31 are
 669          * reserved there).
 670          */
 671         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
 672 }
 673
 674 /**
 675  * ppc440spe_desc_set_dcheck - set CHECK pattern
 676  */
 677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
 678                                 struct ppc440spe_adma_chan *chan, u8 *qword)
 679 {
 680         struct dma_cdb *dma_hw_desc;
 681
 682         switch (chan->device->id) {
 683         case PPC440SPE_DMA0_ID:
 684         case PPC440SPE_DMA1_ID:
 685                 dma_hw_desc = desc->hw_desc;
 686                 iowrite32(qword[0], &dma_hw_desc->sg3l);
 687                 iowrite32(qword[4], &dma_hw_desc->sg3u);
 688                 iowrite32(qword[8], &dma_hw_desc->sg2l);
 689                 iowrite32(qword[12], &dma_hw_desc->sg2u);
 690                 break;
 691         default:
 692                 BUG();
 693         }
 694 }
 695
 696 /**
 697  * ppc440spe_xor_set_link - set link address in xor CB
 698  */
 699 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
 700                                 struct ppc440spe_adma_desc_slot *next_desc)
 701 {
 702         struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
 703
 704         if (unlikely(!next_desc || !(next_desc->phys))) {
 705                 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
 706                         __func__, next_desc,
 707                         next_desc ? next_desc->phys : 0);
 708                 BUG();
 709         }
 710
 711         xor_hw_desc->cbs = 0;
 712         xor_hw_desc->cblal = next_desc->phys;
 713         xor_hw_desc->cblah = 0;
 714         xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
 715 }
 716
 717 /**
 718  * ppc440spe_desc_set_link - set the address of descriptor following this
 719  * descriptor in chain
 720  */
 721 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
 722                                 struct ppc440spe_adma_desc_slot *prev_desc,
 723                                 struct ppc440spe_adma_desc_slot *next_desc)
 724 {
 725         unsigned long flags;
 726         struct ppc440spe_adma_desc_slot *tail = next_desc;
 727
 728         if (unlikely(!prev_desc || !next_desc ||
 729                 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
 730                 /* If previous next is overwritten something is wrong.
 731                  * though we may refetch from append to initiate list
 732                  * processing; in this case - it's ok.
 733                  */
 734                 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
 735                         "prev->hw_next=0x%p\n", __func__, prev_desc,
 736                         next_desc, prev_desc ? prev_desc->hw_next : 0);
 737                 BUG();
 738         }
 739
 740         local_irq_save(flags);
 741
 742         /* do s/w chaining both for DMA and XOR descriptors */
 743         prev_desc->hw_next = next_desc;
 744
 745         switch (chan->device->id) {
 746         case PPC440SPE_DMA0_ID:
 747         case PPC440SPE_DMA1_ID:
 748                 break;
 749         case PPC440SPE_XOR_ID:
 750                 /* bind descriptor to the chain */
 751                 while (tail->hw_next)
 752                         tail = tail->hw_next;
 753                 xor_last_linked = tail;
 754
 755                 if (prev_desc == xor_last_submit)
 756                         /* do not link to the last submitted CB */
 757                         break;
 758                 ppc440spe_xor_set_link(prev_desc, next_desc);
 759                 break;
 760         }
 761
 762         local_irq_restore(flags);
 763 }
 764
 765 /**
 766  * ppc440spe_desc_get_link - get the address of the descriptor that
 767  * follows this one
 768  */
 769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
 770                                         struct ppc440spe_adma_chan *chan)
 771 {
 772         if (!desc->hw_next)
 773                 return 0;
 774
 775         return desc->hw_next->phys;
 776 }
 777
 778 /**
 779  * ppc440spe_desc_is_aligned - check alignment
 780  */
 781 static inline int ppc440spe_desc_is_aligned(
 782         struct ppc440spe_adma_desc_slot *desc, int num_slots)
 783 {
 784         return (desc->idx & (num_slots - 1)) ? 0 : 1;
 785 }
 786
 787 /**
 788  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
 789  * XOR operation
 790  */
 791 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
 792                         int *slots_per_op)
 793 {
 794         int slot_cnt;
 795
 796         /* each XOR descriptor provides up to 16 source operands */
 797         slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
 798
 799         if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
 800                 return slot_cnt;
 801
 802         printk(KERN_ERR "%s: len %d > max %d !!\n",
 803                 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
 804         BUG();
 805         return slot_cnt;
 806 }
 807
 808 /**
 809  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
 810  * DMA2 PQ operation
 811  */
 812 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
 813                 int src_cnt, size_t len)
 814 {
 815         signed long long order = 0;
 816         int state = 0;
 817         int addr_count = 0;
 818         int i;
 819         for (i = 1; i < src_cnt; i++) {
 820                 dma_addr_t cur_addr = srcs[i];
 821                 dma_addr_t old_addr = srcs[i-1];
 822                 switch (state) {
 823                 case 0:
 824                         if (cur_addr == old_addr + len) {
 825                                 /* direct RXOR */
 826                                 order = 1;
 827                                 state = 1;
 828                                 if (i == src_cnt-1)
 829                                         addr_count++;
 830                         } else if (old_addr == cur_addr + len) {
 831                                 /* reverse RXOR */
 832                                 order = -1;
 833                                 state = 1;
 834                                 if (i == src_cnt-1)
 835                                         addr_count++;
 836                         } else {
 837                                 state = 3;
 838                         }
 839                         break;
 840                 case 1:
 841                         if (i == src_cnt-2 || (order == -1
 842                                 && cur_addr != old_addr - len)) {
 843                                 order = 0;
 844                                 state = 0;
 845                                 addr_count++;
 846                         } else if (cur_addr == old_addr + len*order) {
 847                                 state = 2;
 848                                 if (i == src_cnt-1)
 849                                         addr_count++;
 850                         } else if (cur_addr == old_addr + 2*len) {
 851                                 state = 2;
 852                                 if (i == src_cnt-1)
 853                                         addr_count++;
 854                         } else if (cur_addr == old_addr + 3*len) {
 855                                 state = 2;
 856                                 if (i == src_cnt-1)
 857                                         addr_count++;
 858                         } else {
 859                                 order = 0;
 860                                 state = 0;
 861                                 addr_count++;
 862                         }
 863                         break;
 864                 case 2:
 865                         order = 0;
 866                         state = 0;
 867                         addr_count++;
 868                                 break;
 869                 }
 870                 if (state == 3)
 871                         break;
 872         }
 873         if (src_cnt <= 1 || (state != 1 && state != 2)) {
 874                 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
 875                         __func__, src_cnt, state, addr_count, order);
 876                 for (i = 0; i < src_cnt; i++)
 877                         pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
 878                 BUG();
 879         }
 880
 881         return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
 882 }
 883
 884
 885 /******************************************************************************
 886  * ADMA channel low-level routines
 887  ******************************************************************************/
 888
 889 static u32
 890 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
 891 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
 892
 893 /**
 894  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
 895  */
 896 static void ppc440spe_adma_device_clear_eot_status(
 897                                         struct ppc440spe_adma_chan *chan)
 898 {
 899         struct dma_regs *dma_reg;
 900         struct xor_regs *xor_reg;
 901         u8 *p = chan->device->dma_desc_pool_virt;
 902         struct dma_cdb *cdb;
 903         u32 rv, i;
 904
 905         switch (chan->device->id) {
 906         case PPC440SPE_DMA0_ID:
 907         case PPC440SPE_DMA1_ID:
 908                 /* read FIFO to ack */
 909                 dma_reg = chan->device->dma_reg;
 910                 while ((rv = ioread32(&dma_reg->csfpl))) {
 911                         i = rv & DMA_CDB_ADDR_MSK;
 912                         cdb = (struct dma_cdb *)&p[i -
 913                             (u32)chan->device->dma_desc_pool];
 914
 915                         /* Clear opcode to ack. This is necessary for
 916                          * ZeroSum operations only
 917                          */
 918                         cdb->opc = 0;
 919
 920                         if (test_bit(PPC440SPE_RXOR_RUN,
 921                             &ppc440spe_rxor_state)) {
 922                                 /* probably this is a completed RXOR op,
 923                                  * get pointer to CDB using the fact that
 924                                  * physical and virtual addresses of CDB
 925                                  * in pools have the same offsets
 926                                  */
 927                                 if (le32_to_cpu(cdb->sg1u) &
 928                                     DMA_CUED_XOR_BASE) {
 929                                         /* this is a RXOR */
 930                                         clear_bit(PPC440SPE_RXOR_RUN,
 931                                                   &ppc440spe_rxor_state);
 932                                 }
 933                         }
 934
 935                         if (rv & DMA_CDB_STATUS_MSK) {
 936                                 /* ZeroSum check failed
 937                                  */
 938                                 struct ppc440spe_adma_desc_slot *iter;
 939                                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
 940
 941                                 /*
 942                                  * Update the status of corresponding
 943                                  * descriptor.
 944                                  */
 945                                 list_for_each_entry(iter, &chan->chain,
 946                                     chain_node) {
 947                                         if (iter->phys == phys)
 948                                                 break;
 949                                 }
 950                                 /*
 951                                  * if cannot find the corresponding
 952                                  * slot it's a bug
 953                                  */
 954                                 BUG_ON(&iter->chain_node == &chan->chain);
 955
 956                                 if (iter->xor_check_result) {
 957                                         if (test_bit(PPC440SPE_DESC_PCHECK,
 958                                                      &iter->flags)) {
 959                                                 *iter->xor_check_result |=
 960                                                         SUM_CHECK_P_RESULT;
 961                                         } else
 962                                         if (test_bit(PPC440SPE_DESC_QCHECK,
 963                                                      &iter->flags)) {
 964                                                 *iter->xor_check_result |=
 965                                                         SUM_CHECK_Q_RESULT;
 966                                         } else
 967                                                 BUG();
 968                                 }
 969                         }
 970                 }
 971
 972                 rv = ioread32(&dma_reg->dsts);
 973                 if (rv) {
 974                         pr_err("DMA%d err status: 0x%x\n",
 975                                chan->device->id, rv);
 976                         /* write back to clear */
 977                         iowrite32(rv, &dma_reg->dsts);
 978                 }
 979                 break;
 980         case PPC440SPE_XOR_ID:
 981                 /* reset status bits to ack */
 982                 xor_reg = chan->device->xor_reg;
 983                 rv = ioread32be(&xor_reg->sr);
 984                 iowrite32be(rv, &xor_reg->sr);
 985
 986                 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
 987                         if (rv & XOR_IE_RPTIE_BIT) {
 988                                 /* Read PLB Timeout Error.
 989                                  * Try to resubmit the CB
 990                                  */
 991                                 u32 val = ioread32be(&xor_reg->ccbalr);
 992
 993                                 iowrite32be(val, &xor_reg->cblalr);
 994
 995                                 val = ioread32be(&xor_reg->crsr);
 996                                 iowrite32be(val | XOR_CRSR_XAE_BIT,
 997                                             &xor_reg->crsr);
 998                         } else
 999                                 pr_err("XOR ERR 0x%x status\n", rv);
1000                         break;
1001                 }
1002
1003                 /*  if the XORcore is idle, but there are unprocessed CBs
1004                  * then refetch the s/w chain here
1005                  */
1006                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1007                     do_xor_refetch)
1008                         ppc440spe_chan_append(chan);
1009                 break;
1010         }
1011 }
1012
1013 /**
1014  * ppc440spe_chan_is_busy - get the channel status
1015  */
1016 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1017 {
1018         struct dma_regs *dma_reg;
1019         struct xor_regs *xor_reg;
1020         int busy = 0;
1021
1022         switch (chan->device->id) {
1023         case PPC440SPE_DMA0_ID:
1024         case PPC440SPE_DMA1_ID:
1025                 dma_reg = chan->device->dma_reg;
1026                 /*  if command FIFO's head and tail pointers are equal and
1027                  * status tail is the same as command, then channel is free
1028                  */
1029                 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1030                     ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1031                         busy = 1;
1032                 break;
1033         case PPC440SPE_XOR_ID:
1034                 /* use the special status bit for the XORcore
1035                  */
1036                 xor_reg = chan->device->xor_reg;
1037                 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1038                 break;
1039         }
1040
1041         return busy;
1042 }
1043
1044 /**
1045  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1046  */
1047 static void ppc440spe_chan_set_first_xor_descriptor(
1048                                 struct ppc440spe_adma_chan *chan,
1049                                 struct ppc440spe_adma_desc_slot *next_desc)
1050 {
1051         struct xor_regs *xor_reg = chan->device->xor_reg;
1052
1053         if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1054                 printk(KERN_INFO "%s: Warn: XORcore is running "
1055                         "when try to set the first CDB!\n",
1056                         __func__);
1057
1058         xor_last_submit = xor_last_linked = next_desc;
1059
1060         iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1061
1062         iowrite32be(next_desc->phys, &xor_reg->cblalr);
1063         iowrite32be(0, &xor_reg->cblahr);
1064         iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1065                     &xor_reg->cbcr);
1066
1067         chan->hw_chain_inited = 1;
1068 }
1069
1070 /**
1071  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1072  * called with irqs disabled
1073  */
1074 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1075                 struct ppc440spe_adma_desc_slot *desc)
1076 {
1077         u32 pcdb;
1078         struct dma_regs *dma_reg = chan->device->dma_reg;
1079
1080         pcdb = desc->phys;
1081         if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1082                 pcdb |= DMA_CDB_NO_INT;
1083
1084         chan_last_sub[chan->device->id] = desc;
1085
1086         ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1087
1088         iowrite32(pcdb, &dma_reg->cpfpl);
1089 }
1090
1091 /**
1092  * ppc440spe_chan_append - update the h/w chain in the channel
1093  */
1094 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1095 {
1096         struct xor_regs *xor_reg;
1097         struct ppc440spe_adma_desc_slot *iter;
1098         struct xor_cb *xcb;
1099         u32 cur_desc;
1100         unsigned long flags;
1101
1102         local_irq_save(flags);
1103
1104         switch (chan->device->id) {
1105         case PPC440SPE_DMA0_ID:
1106         case PPC440SPE_DMA1_ID:
1107                 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1108
1109                 if (likely(cur_desc)) {
1110                         iter = chan_last_sub[chan->device->id];
1111                         BUG_ON(!iter);
1112                 } else {
1113                         /* first peer */
1114                         iter = chan_first_cdb[chan->device->id];
1115                         BUG_ON(!iter);
1116                         ppc440spe_dma_put_desc(chan, iter);
1117                         chan->hw_chain_inited = 1;
1118                 }
1119
1120                 /* is there something new to append */
1121                 if (!iter->hw_next)
1122                         break;
1123
1124                 /* flush descriptors from the s/w queue to fifo */
1125                 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1126                         ppc440spe_dma_put_desc(chan, iter);
1127                         if (!iter->hw_next)
1128                                 break;
1129                 }
1130                 break;
1131         case PPC440SPE_XOR_ID:
1132                 /* update h/w links and refetch */
1133                 if (!xor_last_submit->hw_next)
1134                         break;
1135
1136                 xor_reg = chan->device->xor_reg;
1137                 /* the last linked CDB has to generate an interrupt
1138                  * that we'd be able to append the next lists to h/w
1139                  * regardless of the XOR engine state at the moment of
1140                  * appending of these next lists
1141                  */
1142                 xcb = xor_last_linked->hw_desc;
1143                 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1144
1145                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1146                         /* XORcore is idle. Refetch now */
1147                         do_xor_refetch = 0;
1148                         ppc440spe_xor_set_link(xor_last_submit,
1149                                 xor_last_submit->hw_next);
1150
1151                         ADMA_LL_DBG(print_cb_list(chan,
1152                                 xor_last_submit->hw_next));
1153
1154                         xor_last_submit = xor_last_linked;
1155                         iowrite32be(ioread32be(&xor_reg->crsr) |
1156                                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1157                                     &xor_reg->crsr);
1158                 } else {
1159                         /* XORcore is running. Refetch later in the handler */
1160                         do_xor_refetch = 1;
1161                 }
1162
1163                 break;
1164         }
1165
1166         local_irq_restore(flags);
1167 }
1168
1169 /**
1170  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1171  */
1172 static u32
1173 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1174 {
1175         struct dma_regs *dma_reg;
1176         struct xor_regs *xor_reg;
1177
1178         if (unlikely(!chan->hw_chain_inited))
1179                 /* h/w descriptor chain is not initialized yet */
1180                 return 0;
1181
1182         switch (chan->device->id) {
1183         case PPC440SPE_DMA0_ID:
1184         case PPC440SPE_DMA1_ID:
1185                 dma_reg = chan->device->dma_reg;
1186                 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1187         case PPC440SPE_XOR_ID:
1188                 xor_reg = chan->device->xor_reg;
1189                 return ioread32be(&xor_reg->ccbalr);
1190         }
1191         return 0;
1192 }
1193
1194 /**
1195  * ppc440spe_chan_run - enable the channel
1196  */
1197 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1198 {
1199         struct xor_regs *xor_reg;
1200
1201         switch (chan->device->id) {
1202         case PPC440SPE_DMA0_ID:
1203         case PPC440SPE_DMA1_ID:
1204                 /* DMAs are always enabled, do nothing */
1205                 break;
1206         case PPC440SPE_XOR_ID:
1207                 /* drain write buffer */
1208                 xor_reg = chan->device->xor_reg;
1209
1210                 /* fetch descriptor pointed to in <link> */
1211                 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1212                             &xor_reg->crsr);
1213                 break;
1214         }
1215 }
1216
1217 /******************************************************************************
1218  * ADMA device level
1219  ******************************************************************************/
1220
1221 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1222 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1223
1224 static dma_cookie_t
1225 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1226
1227 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1228                                     dma_addr_t addr, int index);
1229 static void
1230 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1231                                   dma_addr_t addr, int index);
1232
1233 static void
1234 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1235                            dma_addr_t *paddr, unsigned long flags);
1236 static void
1237 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1238                           dma_addr_t addr, int index);
1239 static void
1240 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1241                                unsigned char mult, int index, int dst_pos);
1242 static void
1243 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1244                                    dma_addr_t paddr, dma_addr_t qaddr);
1245
1246 static struct page *ppc440spe_rxor_srcs[32];
1247
1248 /**
1249  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1250  */
1251 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1252 {
1253         int i, order = 0, state = 0;
1254         int idx = 0;
1255
1256         if (unlikely(!(src_cnt > 1)))
1257                 return 0;
1258
1259         BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1260
1261         /* Skip holes in the source list before checking */
1262         for (i = 0; i < src_cnt; i++) {
1263                 if (!srcs[i])
1264                         continue;
1265                 ppc440spe_rxor_srcs[idx++] = srcs[i];
1266         }
1267         src_cnt = idx;
1268
1269         for (i = 1; i < src_cnt; i++) {
1270                 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1271                 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1272
1273                 switch (state) {
1274                 case 0:
1275                         if (cur_addr == old_addr + len) {
1276                                 /* direct RXOR */
1277                                 order = 1;
1278                                 state = 1;
1279                         } else if (old_addr == cur_addr + len) {
1280                                 /* reverse RXOR */
1281                                 order = -1;
1282                                 state = 1;
1283                         } else
1284                                 goto out;
1285                         break;
1286                 case 1:
1287                         if ((i == src_cnt - 2) ||
1288                             (order == -1 && cur_addr != old_addr - len)) {
1289                                 order = 0;
1290                                 state = 0;
1291                         } else if ((cur_addr == old_addr + len * order) ||
1292                                    (cur_addr == old_addr + 2 * len) ||
1293                                    (cur_addr == old_addr + 3 * len)) {
1294                                 state = 2;
1295                         } else {
1296                                 order = 0;
1297                                 state = 0;
1298                         }
1299                         break;
1300                 case 2:
1301                         order = 0;
1302                         state = 0;
1303                         break;
1304                 }
1305         }
1306
1307 out:
1308         if (state == 1 || state == 2)
1309                 return 1;
1310
1311         return 0;
1312 }
1313
1314 /**
1315  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1316  *      the operation given on this channel. It's assumed that 'chan' is
1317  *      capable to process 'cap' type of operation.
1318  * @chan: channel to use
1319  * @cap: type of transaction
1320  * @dst_lst: array of destination pointers
1321  * @dst_cnt: number of destination operands
1322  * @src_lst: array of source pointers
1323  * @src_cnt: number of source operands
1324  * @src_sz: size of each source operand
1325  */
1326 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1327         enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1328         struct page **src_lst, int src_cnt, size_t src_sz)
1329 {
1330         int ef = 1;
1331
1332         if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1333                 /* If RAID-6 capabilities were not activated don't try
1334                  * to use them
1335                  */
1336                 if (unlikely(!ppc440spe_r6_enabled))
1337                         return -1;
1338         }
1339         /*  In the current implementation of ppc440spe ADMA driver it
1340          * makes sense to pick out only pq case, because it may be
1341          * processed:
1342          * (1) either using Biskup method on DMA2;
1343          * (2) or on DMA0/1.
1344          *  Thus we give a favour to (1) if the sources are suitable;
1345          * else let it be processed on one of the DMA0/1 engines.
1346          *  In the sum_product case where destination is also the
1347          * source process it on DMA0/1 only.
1348          */
1349         if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1350
1351                 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1352                         ef = 0; /* sum_product case, process on DMA0/1 */
1353                 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1354                         ef = 3; /* override (DMA0/1 + idle) */
1355                 else
1356                         ef = 0; /* can't process on DMA2 if !rxor */
1357         }
1358
1359         /* channel idleness increases the priority */
1360         if (likely(ef) &&
1361             !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1362                 ef++;
1363
1364         return ef;
1365 }
1366
1367 struct dma_chan *
1368 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1369         struct page **dst_lst, int dst_cnt, struct page **src_lst,
1370         int src_cnt, size_t src_sz)
1371 {
1372         struct dma_chan *best_chan = NULL;
1373         struct ppc_dma_chan_ref *ref;
1374         int best_rank = -1;
1375
1376         if (unlikely(!src_sz))
1377                 return NULL;
1378         if (src_sz > PAGE_SIZE) {
1379                 /*
1380                  * should a user of the api ever pass > PAGE_SIZE requests
1381                  * we sort out cases where temporary page-sized buffers
1382                  * are used.
1383                  */
1384                 switch (cap) {
1385                 case DMA_PQ:
1386                         if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1387                                 return NULL;
1388                         if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1389                                 return NULL;
1390                         break;
1391                 case DMA_PQ_VAL:
1392                 case DMA_XOR_VAL:
1393                         return NULL;
1394                 default:
1395                         break;
1396                 }
1397         }
1398
1399         list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1400                 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1401                         int rank;
1402
1403                         rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1404                                         dst_cnt, src_lst, src_cnt, src_sz);
1405                         if (rank > best_rank) {
1406                                 best_rank = rank;
1407                                 best_chan = ref->chan;
1408                         }
1409                 }
1410         }
1411
1412         return best_chan;
1413 }
1414 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1415
1416 /**
1417  * ppc440spe_get_group_entry - get group entry with index idx
1418  * @tdesc: is the last allocated slot in the group.
1419  */
1420 static struct ppc440spe_adma_desc_slot *
1421 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1422 {
1423         struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1424         int i = 0;
1425
1426         if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1427                 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1428                         __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1429                 BUG();
1430         }
1431
1432         list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1433                 if (i++ == entry_idx)
1434                         break;
1435         }
1436         return iter;
1437 }
1438
1439 /**
1440  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1441  * @slot: Slot to free
1442  * Caller must hold &ppc440spe_chan->lock while calling this function
1443  */
1444 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1445                                       struct ppc440spe_adma_chan *chan)
1446 {
1447         int stride = slot->slots_per_op;
1448
1449         while (stride--) {
1450                 slot->slots_per_op = 0;
1451                 slot = list_entry(slot->slot_node.next,
1452                                 struct ppc440spe_adma_desc_slot,
1453                                 slot_node);
1454         }
1455 }
1456
1457 /**
1458  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1459  * upon completion
1460  */
1461 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1462                 struct ppc440spe_adma_desc_slot *desc,
1463                 struct ppc440spe_adma_chan *chan,
1464                 dma_cookie_t cookie)
1465 {
1466         BUG_ON(desc->async_tx.cookie < 0);
1467         if (desc->async_tx.cookie > 0) {
1468                 cookie = desc->async_tx.cookie;
1469                 desc->async_tx.cookie = 0;
1470
1471                 dma_descriptor_unmap(&desc->async_tx);
1472                 /* call the callback (must not sleep or submit new
1473                  * operations to this channel)
1474                  */
1475                 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
1476         }
1477
1478         /* run dependent operations */
1479         dma_run_dependencies(&desc->async_tx);
1480
1481         return cookie;
1482 }
1483
1484 /**
1485  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1486  */
1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1488                 struct ppc440spe_adma_chan *chan)
1489 {
1490         /* the client is allowed to attach dependent operations
1491          * until 'ack' is set
1492          */
1493         if (!async_tx_test_ack(&desc->async_tx))
1494                 return 0;
1495
1496         /* leave the last descriptor in the chain
1497          * so we can append to it
1498          */
1499         if (list_is_last(&desc->chain_node, &chan->chain) ||
1500             desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1501                 return 1;
1502
1503         if (chan->device->id != PPC440SPE_XOR_ID) {
1504                 /* our DMA interrupt handler clears opc field of
1505                  * each processed descriptor. For all types of
1506                  * operations except for ZeroSum we do not actually
1507                  * need ack from the interrupt handler. ZeroSum is a
1508                  * special case since the result of this operation
1509                  * is available from the handler only, so if we see
1510                  * such type of descriptor (which is unprocessed yet)
1511                  * then leave it in chain.
1512                  */
1513                 struct dma_cdb *cdb = desc->hw_desc;
1514                 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1515                         return 1;
1516         }
1517
1518         dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1519                 desc->phys, desc->idx, desc->slots_per_op);
1520
1521         list_del(&desc->chain_node);
1522         ppc440spe_adma_free_slots(desc, chan);
1523         return 0;
1524 }
1525
1526 /**
1527  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1528  *      which runs through the channel CDBs list until reach the descriptor
1529  *      currently processed. When routine determines that all CDBs of group
1530  *      are completed then corresponding callbacks (if any) are called and slots
1531  *      are freed.
1532  */
1533 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1534 {
1535         struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1536         dma_cookie_t cookie = 0;
1537         u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1538         int busy = ppc440spe_chan_is_busy(chan);
1539         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1540
1541         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1542                 chan->device->id, __func__);
1543
1544         if (!current_desc) {
1545                 /*  There were no transactions yet, so
1546                  * nothing to clean
1547                  */
1548                 return;
1549         }
1550
1551         /* free completed slots from the chain starting with
1552          * the oldest descriptor
1553          */
1554         list_for_each_entry_safe(iter, _iter, &chan->chain,
1555                                         chain_node) {
1556                 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1557                     "busy: %d this_desc: %#llx next_desc: %#x "
1558                     "cur: %#x ack: %d\n",
1559                     iter->async_tx.cookie, iter->idx, busy, iter->phys,
1560                     ppc440spe_desc_get_link(iter, chan), current_desc,
1561                     async_tx_test_ack(&iter->async_tx));
1562                 prefetch(_iter);
1563                 prefetch(&_iter->async_tx);
1564
1565                 /* do not advance past the current descriptor loaded into the
1566                  * hardware channel,subsequent descriptors are either in process
1567                  * or have not been submitted
1568                  */
1569                 if (seen_current)
1570                         break;
1571
1572                 /* stop the search if we reach the current descriptor and the
1573                  * channel is busy, or if it appears that the current descriptor
1574                  * needs to be re-read (i.e. has been appended to)
1575                  */
1576                 if (iter->phys == current_desc) {
1577                         BUG_ON(seen_current++);
1578                         if (busy || ppc440spe_desc_get_link(iter, chan)) {
1579                                 /* not all descriptors of the group have
1580                                  * been completed; exit.
1581                                  */
1582                                 break;
1583                         }
1584                 }
1585
1586                 /* detect the start of a group transaction */
1587                 if (!slot_cnt && !slots_per_op) {
1588                         slot_cnt = iter->slot_cnt;
1589                         slots_per_op = iter->slots_per_op;
1590                         if (slot_cnt <= slots_per_op) {
1591                                 slot_cnt = 0;
1592                                 slots_per_op = 0;
1593                         }
1594                 }
1595
1596                 if (slot_cnt) {
1597                         if (!group_start)
1598                                 group_start = iter;
1599                         slot_cnt -= slots_per_op;
1600                 }
1601
1602                 /* all the members of a group are complete */
1603                 if (slots_per_op != 0 && slot_cnt == 0) {
1604                         struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1605                         int end_of_chain = 0;
1606
1607                         /* clean up the group */
1608                         slot_cnt = group_start->slot_cnt;
1609                         grp_iter = group_start;
1610                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
1611                                 &chan->chain, chain_node) {
1612
1613                                 cookie = ppc440spe_adma_run_tx_complete_actions(
1614                                         grp_iter, chan, cookie);
1615
1616                                 slot_cnt -= slots_per_op;
1617                                 end_of_chain = ppc440spe_adma_clean_slot(
1618                                     grp_iter, chan);
1619                                 if (end_of_chain && slot_cnt) {
1620                                         /* Should wait for ZeroSum completion */
1621                                         if (cookie > 0)
1622                                                 chan->common.completed_cookie = cookie;
1623                                         return;
1624                                 }
1625
1626                                 if (slot_cnt == 0 || end_of_chain)
1627                                         break;
1628                         }
1629
1630                         /* the group should be complete at this point */
1631                         BUG_ON(slot_cnt);
1632
1633                         slots_per_op = 0;
1634                         group_start = NULL;
1635                         if (end_of_chain)
1636                                 break;
1637                         else
1638                                 continue;
1639                 } else if (slots_per_op) /* wait for group completion */
1640                         continue;
1641
1642                 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1643                     cookie);
1644
1645                 if (ppc440spe_adma_clean_slot(iter, chan))
1646                         break;
1647         }
1648
1649         BUG_ON(!seen_current);
1650
1651         if (cookie > 0) {
1652                 chan->common.completed_cookie = cookie;
1653                 pr_debug("\tcompleted cookie %d\n", cookie);
1654         }
1655
1656 }
1657
1658 /**
1659  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1660  */
1661 static void ppc440spe_adma_tasklet(struct tasklet_struct *t)
1662 {
1663         struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet);
1664
1665         spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1666         __ppc440spe_adma_slot_cleanup(chan);
1667         spin_unlock(&chan->lock);
1668 }
1669
1670 /**
1671  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1672  */
1673 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1674 {
1675         spin_lock_bh(&chan->lock);
1676         __ppc440spe_adma_slot_cleanup(chan);
1677         spin_unlock_bh(&chan->lock);
1678 }
1679
1680 /**
1681  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1682  */
1683 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1684                 struct ppc440spe_adma_chan *chan, int num_slots,
1685                 int slots_per_op)
1686 {
1687         struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1688         struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1689         struct list_head chain = LIST_HEAD_INIT(chain);
1690         int slots_found, retry = 0;
1691
1692
1693         BUG_ON(!num_slots || !slots_per_op);
1694         /* start search from the last allocated descrtiptor
1695          * if a contiguous allocation can not be found start searching
1696          * from the beginning of the list
1697          */
1698 retry:
1699         slots_found = 0;
1700         if (retry == 0)
1701                 iter = chan->last_used;
1702         else
1703                 iter = list_entry(&chan->all_slots,
1704                                   struct ppc440spe_adma_desc_slot,
1705                                   slot_node);
1706         list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1707             slot_node) {
1708                 prefetch(_iter);
1709                 prefetch(&_iter->async_tx);
1710                 if (iter->slots_per_op) {
1711                         slots_found = 0;
1712                         continue;
1713                 }
1714
1715                 /* start the allocation if the slot is correctly aligned */
1716                 if (!slots_found++)
1717                         alloc_start = iter;
1718
1719                 if (slots_found == num_slots) {
1720                         struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1721                         struct ppc440spe_adma_desc_slot *last_used = NULL;
1722
1723                         iter = alloc_start;
1724                         while (num_slots) {
1725                                 int i;
1726                                 /* pre-ack all but the last descriptor */
1727                                 if (num_slots != slots_per_op)
1728                                         async_tx_ack(&iter->async_tx);
1729
1730                                 list_add_tail(&iter->chain_node, &chain);
1731                                 alloc_tail = iter;
1732                                 iter->async_tx.cookie = 0;
1733                                 iter->hw_next = NULL;
1734                                 iter->flags = 0;
1735                                 iter->slot_cnt = num_slots;
1736                                 iter->xor_check_result = NULL;
1737                                 for (i = 0; i < slots_per_op; i++) {
1738                                         iter->slots_per_op = slots_per_op - i;
1739                                         last_used = iter;
1740                                         iter = list_entry(iter->slot_node.next,
1741                                                 struct ppc440spe_adma_desc_slot,
1742                                                 slot_node);
1743                                 }
1744                                 num_slots -= slots_per_op;
1745                         }
1746                         alloc_tail->group_head = alloc_start;
1747                         alloc_tail->async_tx.cookie = -EBUSY;
1748                         list_splice(&chain, &alloc_tail->group_list);
1749                         chan->last_used = last_used;
1750                         return alloc_tail;
1751                 }
1752         }
1753         if (!retry++)
1754                 goto retry;
1755
1756         /* try to free some slots if the allocation fails */
1757         tasklet_schedule(&chan->irq_tasklet);
1758         return NULL;
1759 }
1760
1761 /**
1762  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1763  */
1764 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1765 {
1766         struct ppc440spe_adma_chan *ppc440spe_chan;
1767         struct ppc440spe_adma_desc_slot *slot = NULL;
1768         char *hw_desc;
1769         int i, db_sz;
1770         int init;
1771
1772         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1773         init = ppc440spe_chan->slots_allocated ? 0 : 1;
1774         chan->chan_id = ppc440spe_chan->device->id;
1775
1776         /* Allocate descriptor slots */
1777         i = ppc440spe_chan->slots_allocated;
1778         if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1779                 db_sz = sizeof(struct dma_cdb);
1780         else
1781                 db_sz = sizeof(struct xor_cb);
1782
1783         for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1784                 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1785                                GFP_KERNEL);
1786                 if (!slot) {
1787                         printk(KERN_INFO "SPE ADMA Channel only initialized"
1788                                 " %d descriptor slots", i--);
1789                         break;
1790                 }
1791
1792                 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1793                 slot->hw_desc = (void *) &hw_desc[i * db_sz];
1794                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
1795                 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1796                 INIT_LIST_HEAD(&slot->chain_node);
1797                 INIT_LIST_HEAD(&slot->slot_node);
1798                 INIT_LIST_HEAD(&slot->group_list);
1799                 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1800                 slot->idx = i;
1801
1802                 spin_lock_bh(&ppc440spe_chan->lock);
1803                 ppc440spe_chan->slots_allocated++;
1804                 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1805                 spin_unlock_bh(&ppc440spe_chan->lock);
1806         }
1807
1808         if (i && !ppc440spe_chan->last_used) {
1809                 ppc440spe_chan->last_used =
1810                         list_entry(ppc440spe_chan->all_slots.next,
1811                                 struct ppc440spe_adma_desc_slot,
1812                                 slot_node);
1813         }
1814
1815         dev_dbg(ppc440spe_chan->device->common.dev,
1816                 "ppc440spe adma%d: allocated %d descriptor slots\n",
1817                 ppc440spe_chan->device->id, i);
1818
1819         /* initialize the channel and the chain with a null operation */
1820         if (init) {
1821                 switch (ppc440spe_chan->device->id) {
1822                 case PPC440SPE_DMA0_ID:
1823                 case PPC440SPE_DMA1_ID:
1824                         ppc440spe_chan->hw_chain_inited = 0;
1825                         /* Use WXOR for self-testing */
1826                         if (!ppc440spe_r6_tchan)
1827                                 ppc440spe_r6_tchan = ppc440spe_chan;
1828                         break;
1829                 case PPC440SPE_XOR_ID:
1830                         ppc440spe_chan_start_null_xor(ppc440spe_chan);
1831                         break;
1832                 default:
1833                         BUG();
1834                 }
1835                 ppc440spe_chan->needs_unmap = 1;
1836         }
1837
1838         return (i > 0) ? i : -ENOMEM;
1839 }
1840
1841 /**
1842  * ppc440spe_rxor_set_region_data -
1843  */
1844 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1845         u8 xor_arg_no, u32 mask)
1846 {
1847         struct xor_cb *xcb = desc->hw_desc;
1848
1849         xcb->ops[xor_arg_no].h |= mask;
1850 }
1851
1852 /**
1853  * ppc440spe_rxor_set_src -
1854  */
1855 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1856         u8 xor_arg_no, dma_addr_t addr)
1857 {
1858         struct xor_cb *xcb = desc->hw_desc;
1859
1860         xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1861         xcb->ops[xor_arg_no].l = addr;
1862 }
1863
1864 /**
1865  * ppc440spe_rxor_set_mult -
1866  */
1867 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1868         u8 xor_arg_no, u8 idx, u8 mult)
1869 {
1870         struct xor_cb *xcb = desc->hw_desc;
1871
1872         xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1873 }
1874
1875 /**
1876  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1877  *      has been achieved
1878  */
1879 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1880 {
1881         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1882                 chan->device->id, chan->pending);
1883
1884         if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1885                 chan->pending = 0;
1886                 ppc440spe_chan_append(chan);
1887         }
1888 }
1889
1890 /**
1891  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1892  *      (it's not necessary that descriptors will be submitted to the h/w
1893  *      chains too right now)
1894  */
1895 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1896 {
1897         struct ppc440spe_adma_desc_slot *sw_desc;
1898         struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1899         struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1900         int slot_cnt;
1901         int slots_per_op;
1902         dma_cookie_t cookie;
1903
1904         sw_desc = tx_to_ppc440spe_adma_slot(tx);
1905
1906         group_start = sw_desc->group_head;
1907         slot_cnt = group_start->slot_cnt;
1908         slots_per_op = group_start->slots_per_op;
1909
1910         spin_lock_bh(&chan->lock);
1911         cookie = dma_cookie_assign(tx);
1912
1913         if (unlikely(list_empty(&chan->chain))) {
1914                 /* first peer */
1915                 list_splice_init(&sw_desc->group_list, &chan->chain);
1916                 chan_first_cdb[chan->device->id] = group_start;
1917         } else {
1918                 /* isn't first peer, bind CDBs to chain */
1919                 old_chain_tail = list_entry(chan->chain.prev,
1920                                         struct ppc440spe_adma_desc_slot,
1921                                         chain_node);
1922                 list_splice_init(&sw_desc->group_list,
1923                     &old_chain_tail->chain_node);
1924                 /* fix up the hardware chain */
1925                 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1926         }
1927
1928         /* increment the pending count by the number of operations */
1929         chan->pending += slot_cnt / slots_per_op;
1930         ppc440spe_adma_check_threshold(chan);
1931         spin_unlock_bh(&chan->lock);
1932
1933         dev_dbg(chan->device->common.dev,
1934                 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1935                 chan->device->id, __func__,
1936                 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1937
1938         return cookie;
1939 }
1940
1941 /**
1942  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1943  */
1944 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1945                 struct dma_chan *chan, unsigned long flags)
1946 {
1947         struct ppc440spe_adma_chan *ppc440spe_chan;
1948         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1949         int slot_cnt, slots_per_op;
1950
1951         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1952
1953         dev_dbg(ppc440spe_chan->device->common.dev,
1954                 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1955                 __func__);
1956
1957         spin_lock_bh(&ppc440spe_chan->lock);
1958         slot_cnt = slots_per_op = 1;
1959         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1960                         slots_per_op);
1961         if (sw_desc) {
1962                 group_start = sw_desc->group_head;
1963                 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1964                 group_start->unmap_len = 0;
1965                 sw_desc->async_tx.flags = flags;
1966         }
1967         spin_unlock_bh(&ppc440spe_chan->lock);
1968
1969         return sw_desc ? &sw_desc->async_tx : NULL;
1970 }
1971
1972 /**
1973  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1974  */
1975 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1976                 struct dma_chan *chan, dma_addr_t dma_dest,
1977                 dma_addr_t dma_src, size_t len, unsigned long flags)
1978 {
1979         struct ppc440spe_adma_chan *ppc440spe_chan;
1980         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1981         int slot_cnt, slots_per_op;
1982
1983         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1984
1985         if (unlikely(!len))
1986                 return NULL;
1987
1988         BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1989
1990         spin_lock_bh(&ppc440spe_chan->lock);
1991
1992         dev_dbg(ppc440spe_chan->device->common.dev,
1993                 "ppc440spe adma%d: %s len: %u int_en %d\n",
1994                 ppc440spe_chan->device->id, __func__, len,
1995                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
1996         slot_cnt = slots_per_op = 1;
1997         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1998                 slots_per_op);
1999         if (sw_desc) {
2000                 group_start = sw_desc->group_head;
2001                 ppc440spe_desc_init_memcpy(group_start, flags);
2002                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2003                 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2004                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2005                 sw_desc->unmap_len = len;
2006                 sw_desc->async_tx.flags = flags;
2007         }
2008         spin_unlock_bh(&ppc440spe_chan->lock);
2009
2010         return sw_desc ? &sw_desc->async_tx : NULL;
2011 }
2012
2013 /**
2014  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2015  */
2016 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2017                 struct dma_chan *chan, dma_addr_t dma_dest,
2018                 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2019                 unsigned long flags)
2020 {
2021         struct ppc440spe_adma_chan *ppc440spe_chan;
2022         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2023         int slot_cnt, slots_per_op;
2024
2025         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2026
2027         ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2028                                      dma_dest, dma_src, src_cnt));
2029         if (unlikely(!len))
2030                 return NULL;
2031         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2032
2033         dev_dbg(ppc440spe_chan->device->common.dev,
2034                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2035                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2036                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2037
2038         spin_lock_bh(&ppc440spe_chan->lock);
2039         slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2040         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2041                         slots_per_op);
2042         if (sw_desc) {
2043                 group_start = sw_desc->group_head;
2044                 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2045                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2046                 while (src_cnt--)
2047                         ppc440spe_adma_memcpy_xor_set_src(group_start,
2048                                 dma_src[src_cnt], src_cnt);
2049                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2050                 sw_desc->unmap_len = len;
2051                 sw_desc->async_tx.flags = flags;
2052         }
2053         spin_unlock_bh(&ppc440spe_chan->lock);
2054
2055         return sw_desc ? &sw_desc->async_tx : NULL;
2056 }
2057
2058 static inline void
2059 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2060                                 int src_cnt);
2061 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2062
2063 /**
2064  * ppc440spe_adma_init_dma2rxor_slot -
2065  */
2066 static void ppc440spe_adma_init_dma2rxor_slot(
2067                 struct ppc440spe_adma_desc_slot *desc,
2068                 dma_addr_t *src, int src_cnt)
2069 {
2070         int i;
2071
2072         /* initialize CDB */
2073         for (i = 0; i < src_cnt; i++) {
2074                 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2075                                                  desc->src_cnt, (u32)src[i]);
2076         }
2077 }
2078
2079 /**
2080  * ppc440spe_dma01_prep_mult -
2081  * for Q operation where destination is also the source
2082  */
2083 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2084                 struct ppc440spe_adma_chan *ppc440spe_chan,
2085                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2086                 const unsigned char *scf, size_t len, unsigned long flags)
2087 {
2088         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2089         unsigned long op = 0;
2090         int slot_cnt;
2091
2092         set_bit(PPC440SPE_DESC_WXOR, &op);
2093         slot_cnt = 2;
2094
2095         spin_lock_bh(&ppc440spe_chan->lock);
2096
2097         /* use WXOR, each descriptor occupies one slot */
2098         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2099         if (sw_desc) {
2100                 struct ppc440spe_adma_chan *chan;
2101                 struct ppc440spe_adma_desc_slot *iter;
2102                 struct dma_cdb *hw_desc;
2103
2104                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2105                 set_bits(op, &sw_desc->flags);
2106                 sw_desc->src_cnt = src_cnt;
2107                 sw_desc->dst_cnt = dst_cnt;
2108                 /* First descriptor, zero data in the destination and copy it
2109                  * to q page using MULTICAST transfer.
2110                  */
2111                 iter = list_first_entry(&sw_desc->group_list,
2112                                         struct ppc440spe_adma_desc_slot,
2113                                         chain_node);
2114                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2115                 /* set 'next' pointer */
2116                 iter->hw_next = list_entry(iter->chain_node.next,
2117                                            struct ppc440spe_adma_desc_slot,
2118                                            chain_node);
2119                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2120                 hw_desc = iter->hw_desc;
2121                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2122
2123                 ppc440spe_desc_set_dest_addr(iter, chan,
2124                                              DMA_CUED_XOR_BASE, dst[0], 0);
2125                 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2126                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2127                                             src[0]);
2128                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2129                 iter->unmap_len = len;
2130
2131                 /*
2132                  * Second descriptor, multiply data from the q page
2133                  * and store the result in real destination.
2134                  */
2135                 iter = list_first_entry(&iter->chain_node,
2136                                         struct ppc440spe_adma_desc_slot,
2137                                         chain_node);
2138                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2139                 iter->hw_next = NULL;
2140                 if (flags & DMA_PREP_INTERRUPT)
2141                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2142                 else
2143                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2144
2145                 hw_desc = iter->hw_desc;
2146                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2147                 ppc440spe_desc_set_src_addr(iter, chan, 0,
2148                                             DMA_CUED_XOR_HB, dst[1]);
2149                 ppc440spe_desc_set_dest_addr(iter, chan,
2150                                              DMA_CUED_XOR_BASE, dst[0], 0);
2151
2152                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2153                                             DMA_CDB_SG_DST1, scf[0]);
2154                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2155                 iter->unmap_len = len;
2156                 sw_desc->async_tx.flags = flags;
2157         }
2158
2159         spin_unlock_bh(&ppc440spe_chan->lock);
2160
2161         return sw_desc;
2162 }
2163
2164 /**
2165  * ppc440spe_dma01_prep_sum_product -
2166  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2167  * the source.
2168  */
2169 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2170                 struct ppc440spe_adma_chan *ppc440spe_chan,
2171                 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2172                 const unsigned char *scf, size_t len, unsigned long flags)
2173 {
2174         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2175         unsigned long op = 0;
2176         int slot_cnt;
2177
2178         set_bit(PPC440SPE_DESC_WXOR, &op);
2179         slot_cnt = 3;
2180
2181         spin_lock_bh(&ppc440spe_chan->lock);
2182
2183         /* WXOR, each descriptor occupies one slot */
2184         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2185         if (sw_desc) {
2186                 struct ppc440spe_adma_chan *chan;
2187                 struct ppc440spe_adma_desc_slot *iter;
2188                 struct dma_cdb *hw_desc;
2189
2190                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2191                 set_bits(op, &sw_desc->flags);
2192                 sw_desc->src_cnt = src_cnt;
2193                 sw_desc->dst_cnt = 1;
2194                 /* 1st descriptor, src[1] data to q page and zero destination */
2195                 iter = list_first_entry(&sw_desc->group_list,
2196                                         struct ppc440spe_adma_desc_slot,
2197                                         chain_node);
2198                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2199                 iter->hw_next = list_entry(iter->chain_node.next,
2200                                            struct ppc440spe_adma_desc_slot,
2201                                            chain_node);
2202                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2203                 hw_desc = iter->hw_desc;
2204                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2205
2206                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2207                                              *dst, 0);
2208                 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2209                                              ppc440spe_chan->qdest, 1);
2210                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2211                                             src[1]);
2212                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2213                 iter->unmap_len = len;
2214
2215                 /* 2nd descriptor, multiply src[1] data and store the
2216                  * result in destination */
2217                 iter = list_first_entry(&iter->chain_node,
2218                                         struct ppc440spe_adma_desc_slot,
2219                                         chain_node);
2220                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2221                 /* set 'next' pointer */
2222                 iter->hw_next = list_entry(iter->chain_node.next,
2223                                            struct ppc440spe_adma_desc_slot,
2224                                            chain_node);
2225                 if (flags & DMA_PREP_INTERRUPT)
2226                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2227                 else
2228                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2229
2230                 hw_desc = iter->hw_desc;
2231                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2232                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2233                                             ppc440spe_chan->qdest);
2234                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2235                                              *dst, 0);
2236                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2237                                             DMA_CDB_SG_DST1, scf[1]);
2238                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2239                 iter->unmap_len = len;
2240
2241                 /*
2242                  * 3rd descriptor, multiply src[0] data and xor it
2243                  * with destination
2244                  */
2245                 iter = list_first_entry(&iter->chain_node,
2246                                         struct ppc440spe_adma_desc_slot,
2247                                         chain_node);
2248                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2249                 iter->hw_next = NULL;
2250                 if (flags & DMA_PREP_INTERRUPT)
2251                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2252                 else
2253                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2254
2255                 hw_desc = iter->hw_desc;
2256                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2257                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2258                                             src[0]);
2259                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2260                                              *dst, 0);
2261                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2262                                             DMA_CDB_SG_DST1, scf[0]);
2263                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2264                 iter->unmap_len = len;
2265                 sw_desc->async_tx.flags = flags;
2266         }
2267
2268         spin_unlock_bh(&ppc440spe_chan->lock);
2269
2270         return sw_desc;
2271 }
2272
2273 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2274                 struct ppc440spe_adma_chan *ppc440spe_chan,
2275                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2276                 const unsigned char *scf, size_t len, unsigned long flags)
2277 {
2278         int slot_cnt;
2279         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2280         unsigned long op = 0;
2281         unsigned char mult = 1;
2282
2283         pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2284                  __func__, dst_cnt, src_cnt, len);
2285         /*  select operations WXOR/RXOR depending on the
2286          * source addresses of operators and the number
2287          * of destinations (RXOR support only Q-parity calculations)
2288          */
2289         set_bit(PPC440SPE_DESC_WXOR, &op);
2290         if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2291                 /* no active RXOR;
2292                  * do RXOR if:
2293                  * - there are more than 1 source,
2294                  * - len is aligned on 512-byte boundary,
2295                  * - source addresses fit to one of 4 possible regions.
2296                  */
2297                 if (src_cnt > 1 &&
2298                     !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2299                     (src[0] + len) == src[1]) {
2300                         /* may do RXOR R1 R2 */
2301                         set_bit(PPC440SPE_DESC_RXOR, &op);
2302                         if (src_cnt != 2) {
2303                                 /* may try to enhance region of RXOR */
2304                                 if ((src[1] + len) == src[2]) {
2305                                         /* do RXOR R1 R2 R3 */
2306                                         set_bit(PPC440SPE_DESC_RXOR123,
2307                                                 &op);
2308                                 } else if ((src[1] + len * 2) == src[2]) {
2309                                         /* do RXOR R1 R2 R4 */
2310                                         set_bit(PPC440SPE_DESC_RXOR124, &op);
2311                                 } else if ((src[1] + len * 3) == src[2]) {
2312                                         /* do RXOR R1 R2 R5 */
2313                                         set_bit(PPC440SPE_DESC_RXOR125,
2314                                                 &op);
2315                                 } else {
2316                                         /* do RXOR R1 R2 */
2317                                         set_bit(PPC440SPE_DESC_RXOR12,
2318                                                 &op);
2319                                 }
2320                         } else {
2321                                 /* do RXOR R1 R2 */
2322                                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2323                         }
2324                 }
2325
2326                 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2327                         /* can not do this operation with RXOR */
2328                         clear_bit(PPC440SPE_RXOR_RUN,
2329                                 &ppc440spe_rxor_state);
2330                 } else {
2331                         /* can do; set block size right now */
2332                         ppc440spe_desc_set_rxor_block_size(len);
2333                 }
2334         }
2335
2336         /* Number of necessary slots depends on operation type selected */
2337         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2338                 /*  This is a WXOR only chain. Need descriptors for each
2339                  * source to GF-XOR them with WXOR, and need descriptors
2340                  * for each destination to zero them with WXOR
2341                  */
2342                 slot_cnt = src_cnt;
2343
2344                 if (flags & DMA_PREP_ZERO_P) {
2345                         slot_cnt++;
2346                         set_bit(PPC440SPE_ZERO_P, &op);
2347                 }
2348                 if (flags & DMA_PREP_ZERO_Q) {
2349                         slot_cnt++;
2350                         set_bit(PPC440SPE_ZERO_Q, &op);
2351                 }
2352         } else {
2353                 /*  Need 1/2 descriptor for RXOR operation, and
2354                  * need (src_cnt - (2 or 3)) for WXOR of sources
2355                  * remained (if any)
2356                  */
2357                 slot_cnt = dst_cnt;
2358
2359                 if (flags & DMA_PREP_ZERO_P)
2360                         set_bit(PPC440SPE_ZERO_P, &op);
2361                 if (flags & DMA_PREP_ZERO_Q)
2362                         set_bit(PPC440SPE_ZERO_Q, &op);
2363
2364                 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2365                         slot_cnt += src_cnt - 2;
2366                 else
2367                         slot_cnt += src_cnt - 3;
2368
2369                 /*  Thus we have either RXOR only chain or
2370                  * mixed RXOR/WXOR
2371                  */
2372                 if (slot_cnt == dst_cnt)
2373                         /* RXOR only chain */
2374                         clear_bit(PPC440SPE_DESC_WXOR, &op);
2375         }
2376
2377         spin_lock_bh(&ppc440spe_chan->lock);
2378         /* for both RXOR/WXOR each descriptor occupies one slot */
2379         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2380         if (sw_desc) {
2381                 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2382                                 flags, op);
2383
2384                 /* setup dst/src/mult */
2385                 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2386                          __func__, dst[0], dst[1]);
2387                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2388                 while (src_cnt--) {
2389                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2390                                                   src_cnt);
2391
2392                         /* NOTE: "Multi = 0 is equivalent to = 1" as it
2393                          * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2394                          * doesn't work for RXOR with DMA0/1! Instead, multi=0
2395                          * leads to zeroing source data after RXOR.
2396                          * So, for P case set-up mult=1 explicitly.
2397                          */
2398                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2399                                 mult = scf[src_cnt];
2400                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2401                                 mult, src_cnt,  dst_cnt - 1);
2402                 }
2403
2404                 /* Setup byte count foreach slot just allocated */
2405                 sw_desc->async_tx.flags = flags;
2406                 list_for_each_entry(iter, &sw_desc->group_list,
2407                                 chain_node) {
2408                         ppc440spe_desc_set_byte_count(iter,
2409                                 ppc440spe_chan, len);
2410                         iter->unmap_len = len;
2411                 }
2412         }
2413         spin_unlock_bh(&ppc440spe_chan->lock);
2414
2415         return sw_desc;
2416 }
2417
2418 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2419                 struct ppc440spe_adma_chan *ppc440spe_chan,
2420                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2421                 const unsigned char *scf, size_t len, unsigned long flags)
2422 {
2423         int slot_cnt, descs_per_op;
2424         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2425         unsigned long op = 0;
2426         unsigned char mult = 1;
2427
2428         BUG_ON(!dst_cnt);
2429         /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2430                  __func__, dst_cnt, src_cnt, len);*/
2431
2432         spin_lock_bh(&ppc440spe_chan->lock);
2433         descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2434         if (descs_per_op < 0) {
2435                 spin_unlock_bh(&ppc440spe_chan->lock);
2436                 return NULL;
2437         }
2438
2439         /* depending on number of sources we have 1 or 2 RXOR chains */
2440         slot_cnt = descs_per_op * dst_cnt;
2441
2442         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2443         if (sw_desc) {
2444                 op = slot_cnt;
2445                 sw_desc->async_tx.flags = flags;
2446                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2447                         ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2448                                 --op ? 0 : flags);
2449                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2450                                 len);
2451                         iter->unmap_len = len;
2452
2453                         ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2454                         iter->rxor_cursor.len = len;
2455                         iter->descs_per_op = descs_per_op;
2456                 }
2457                 op = 0;
2458                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2459                         op++;
2460                         if (op % descs_per_op == 0)
2461                                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2462                                                                   src_cnt);
2463                         if (likely(!list_is_last(&iter->chain_node,
2464                                                  &sw_desc->group_list))) {
2465                                 /* set 'next' pointer */
2466                                 iter->hw_next =
2467                                         list_entry(iter->chain_node.next,
2468                                                 struct ppc440spe_adma_desc_slot,
2469                                                 chain_node);
2470                                 ppc440spe_xor_set_link(iter, iter->hw_next);
2471                         } else {
2472                                 /* this is the last descriptor. */
2473                                 iter->hw_next = NULL;
2474                         }
2475                 }
2476
2477                 /* fixup head descriptor */
2478                 sw_desc->dst_cnt = dst_cnt;
2479                 if (flags & DMA_PREP_ZERO_P)
2480                         set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2481                 if (flags & DMA_PREP_ZERO_Q)
2482                         set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2483
2484                 /* setup dst/src/mult */
2485                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2486
2487                 while (src_cnt--) {
2488                         /* handle descriptors (if dst_cnt == 2) inside
2489                          * the ppc440spe_adma_pq_set_srcxxx() functions
2490                          */
2491                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2492                                                   src_cnt);
2493                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2494                                 mult = scf[src_cnt];
2495                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2496                                         mult, src_cnt, dst_cnt - 1);
2497                 }
2498         }
2499         spin_unlock_bh(&ppc440spe_chan->lock);
2500         ppc440spe_desc_set_rxor_block_size(len);
2501         return sw_desc;
2502 }
2503
2504 /**
2505  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2506  */
2507 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2508                 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2509                 unsigned int src_cnt, const unsigned char *scf,
2510                 size_t len, unsigned long flags)
2511 {
2512         struct ppc440spe_adma_chan *ppc440spe_chan;
2513         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2514         int dst_cnt = 0;
2515
2516         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2517
2518         ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2519                                     dst, src, src_cnt));
2520         BUG_ON(!len);
2521         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2522         BUG_ON(!src_cnt);
2523
2524         if (src_cnt == 1 && dst[1] == src[0]) {
2525                 dma_addr_t dest[2];
2526
2527                 /* dst[1] is real destination (Q) */
2528                 dest[0] = dst[1];
2529                 /* this is the page to multicast source data to */
2530                 dest[1] = ppc440spe_chan->qdest;
2531                 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2532                                 dest, 2, src, src_cnt, scf, len, flags);
2533                 return sw_desc ? &sw_desc->async_tx : NULL;
2534         }
2535
2536         if (src_cnt == 2 && dst[1] == src[1]) {
2537                 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2538                                         &dst[1], src, 2, scf, len, flags);
2539                 return sw_desc ? &sw_desc->async_tx : NULL;
2540         }
2541
2542         if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2543                 BUG_ON(!dst[0]);
2544                 dst_cnt++;
2545                 flags |= DMA_PREP_ZERO_P;
2546         }
2547
2548         if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2549                 BUG_ON(!dst[1]);
2550                 dst_cnt++;
2551                 flags |= DMA_PREP_ZERO_Q;
2552         }
2553
2554         BUG_ON(!dst_cnt);
2555
2556         dev_dbg(ppc440spe_chan->device->common.dev,
2557                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2558                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2559                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2560
2561         switch (ppc440spe_chan->device->id) {
2562         case PPC440SPE_DMA0_ID:
2563         case PPC440SPE_DMA1_ID:
2564                 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2565                                 dst, dst_cnt, src, src_cnt, scf,
2566                                 len, flags);
2567                 break;
2568
2569         case PPC440SPE_XOR_ID:
2570                 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2571                                 dst, dst_cnt, src, src_cnt, scf,
2572                                 len, flags);
2573                 break;
2574         }
2575
2576         return sw_desc ? &sw_desc->async_tx : NULL;
2577 }
2578
2579 /**
2580  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2581  * a PQ_ZERO_SUM operation
2582  */
2583 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2584                 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2585                 unsigned int src_cnt, const unsigned char *scf, size_t len,
2586                 enum sum_check_flags *pqres, unsigned long flags)
2587 {
2588         struct ppc440spe_adma_chan *ppc440spe_chan;
2589         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2590         dma_addr_t pdest, qdest;
2591         int slot_cnt, slots_per_op, idst, dst_cnt;
2592
2593         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2594
2595         if (flags & DMA_PREP_PQ_DISABLE_P)
2596                 pdest = 0;
2597         else
2598                 pdest = pq[0];
2599
2600         if (flags & DMA_PREP_PQ_DISABLE_Q)
2601                 qdest = 0;
2602         else
2603                 qdest = pq[1];
2604
2605         ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2606                                             src, src_cnt, scf));
2607
2608         /* Always use WXOR for P/Q calculations (two destinations).
2609          * Need 1 or 2 extra slots to verify results are zero.
2610          */
2611         idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2612
2613         /* One additional slot per destination to clone P/Q
2614          * before calculation (we have to preserve destinations).
2615          */
2616         slot_cnt = src_cnt + dst_cnt * 2;
2617         slots_per_op = 1;
2618
2619         spin_lock_bh(&ppc440spe_chan->lock);
2620         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2621                                              slots_per_op);
2622         if (sw_desc) {
2623                 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2624
2625                 /* Setup byte count for each slot just allocated */
2626                 sw_desc->async_tx.flags = flags;
2627                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2628                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2629                                                       len);
2630                         iter->unmap_len = len;
2631                 }
2632
2633                 if (pdest) {
2634                         struct dma_cdb *hw_desc;
2635                         struct ppc440spe_adma_chan *chan;
2636
2637                         iter = sw_desc->group_head;
2638                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2639                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2640                         iter->hw_next = list_entry(iter->chain_node.next,
2641                                                 struct ppc440spe_adma_desc_slot,
2642                                                 chain_node);
2643                         hw_desc = iter->hw_desc;
2644                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2645                         iter->src_cnt = 0;
2646                         iter->dst_cnt = 0;
2647                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2648                                                      ppc440spe_chan->pdest, 0);
2649                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2650                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2651                                                       len);
2652                         iter->unmap_len = 0;
2653                         /* override pdest to preserve original P */
2654                         pdest = ppc440spe_chan->pdest;
2655                 }
2656                 if (qdest) {
2657                         struct dma_cdb *hw_desc;
2658                         struct ppc440spe_adma_chan *chan;
2659
2660                         iter = list_first_entry(&sw_desc->group_list,
2661                                                 struct ppc440spe_adma_desc_slot,
2662                                                 chain_node);
2663                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2664
2665                         if (pdest) {
2666                                 iter = list_entry(iter->chain_node.next,
2667                                                 struct ppc440spe_adma_desc_slot,
2668                                                 chain_node);
2669                         }
2670
2671                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2672                         iter->hw_next = list_entry(iter->chain_node.next,
2673                                                 struct ppc440spe_adma_desc_slot,
2674                                                 chain_node);
2675                         hw_desc = iter->hw_desc;
2676                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2677                         iter->src_cnt = 0;
2678                         iter->dst_cnt = 0;
2679                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2680                                                      ppc440spe_chan->qdest, 0);
2681                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2682                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2683                                                       len);
2684                         iter->unmap_len = 0;
2685                         /* override qdest to preserve original Q */
2686                         qdest = ppc440spe_chan->qdest;
2687                 }
2688
2689                 /* Setup destinations for P/Q ops */
2690                 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2691
2692                 /* Setup zero QWORDs into DCHECK CDBs */
2693                 idst = dst_cnt;
2694                 list_for_each_entry_reverse(iter, &sw_desc->group_list,
2695                                             chain_node) {
2696                         /*
2697                          * The last CDB corresponds to Q-parity check,
2698                          * the one before last CDB corresponds
2699                          * P-parity check
2700                          */
2701                         if (idst == DMA_DEST_MAX_NUM) {
2702                                 if (idst == dst_cnt) {
2703                                         set_bit(PPC440SPE_DESC_QCHECK,
2704                                                 &iter->flags);
2705                                 } else {
2706                                         set_bit(PPC440SPE_DESC_PCHECK,
2707                                                 &iter->flags);
2708                                 }
2709                         } else {
2710                                 if (qdest) {
2711                                         set_bit(PPC440SPE_DESC_QCHECK,
2712                                                 &iter->flags);
2713                                 } else {
2714                                         set_bit(PPC440SPE_DESC_PCHECK,
2715                                                 &iter->flags);
2716                                 }
2717                         }
2718                         iter->xor_check_result = pqres;
2719
2720                         /*
2721                          * set it to zero, if check fail then result will
2722                          * be updated
2723                          */
2724                         *iter->xor_check_result = 0;
2725                         ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2726                                 ppc440spe_qword);
2727
2728                         if (!(--dst_cnt))
2729                                 break;
2730                 }
2731
2732                 /* Setup sources and mults for P/Q ops */
2733                 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2734                                                      chain_node) {
2735                         struct ppc440spe_adma_chan *chan;
2736                         u32 mult_dst;
2737
2738                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2739                         ppc440spe_desc_set_src_addr(iter, chan, 0,
2740                                                     DMA_CUED_XOR_HB,
2741                                                     src[src_cnt - 1]);
2742                         if (qdest) {
2743                                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2744                                                            DMA_CDB_SG_DST1;
2745                                 ppc440spe_desc_set_src_mult(iter, chan,
2746                                                             DMA_CUED_MULT1_OFF,
2747                                                             mult_dst,
2748                                                             scf[src_cnt - 1]);
2749                         }
2750                         if (!(--src_cnt))
2751                                 break;
2752                 }
2753         }
2754         spin_unlock_bh(&ppc440spe_chan->lock);
2755         return sw_desc ? &sw_desc->async_tx : NULL;
2756 }
2757
2758 /**
2759  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2760  * XOR ZERO_SUM operation
2761  */
2762 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2763                 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2764                 size_t len, enum sum_check_flags *result, unsigned long flags)
2765 {
2766         struct dma_async_tx_descriptor *tx;
2767         dma_addr_t pq[2];
2768
2769         /* validate P, disable Q */
2770         pq[0] = src[0];
2771         pq[1] = 0;
2772         flags |= DMA_PREP_PQ_DISABLE_Q;
2773
2774         tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2775                                                 src_cnt - 1, 0, len,
2776                                                 result, flags);
2777         return tx;
2778 }
2779
2780 /**
2781  * ppc440spe_adma_set_dest - set destination address into descriptor
2782  */
2783 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2784                 dma_addr_t addr, int index)
2785 {
2786         struct ppc440spe_adma_chan *chan;
2787
2788         BUG_ON(index >= sw_desc->dst_cnt);
2789
2790         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2791
2792         switch (chan->device->id) {
2793         case PPC440SPE_DMA0_ID:
2794         case PPC440SPE_DMA1_ID:
2795                 /* to do: support transfers lengths >
2796                  * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2797                  */
2798                 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2799                         chan, 0, addr, index);
2800                 break;
2801         case PPC440SPE_XOR_ID:
2802                 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2803                 ppc440spe_desc_set_dest_addr(sw_desc,
2804                         chan, 0, addr, index);
2805                 break;
2806         }
2807 }
2808
2809 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2810                 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2811 {
2812         /*  To clear destinations update the descriptor
2813          * (P or Q depending on index) as follows:
2814          * addr is destination (0 corresponds to SG2):
2815          */
2816         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2817
2818         /* ... and the addr is source: */
2819         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2820
2821         /* addr is always SG2 then the mult is always DST1 */
2822         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2823                                     DMA_CDB_SG_DST1, 1);
2824 }
2825
2826 /**
2827  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2828  * for the PQXOR operation
2829  */
2830 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2831                 dma_addr_t *addrs, unsigned long flags)
2832 {
2833         struct ppc440spe_adma_desc_slot *iter;
2834         struct ppc440spe_adma_chan *chan;
2835         dma_addr_t paddr, qaddr;
2836         dma_addr_t addr = 0, ppath, qpath;
2837         int index = 0, i;
2838
2839         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2840
2841         if (flags & DMA_PREP_PQ_DISABLE_P)
2842                 paddr = 0;
2843         else
2844                 paddr = addrs[0];
2845
2846         if (flags & DMA_PREP_PQ_DISABLE_Q)
2847                 qaddr = 0;
2848         else
2849                 qaddr = addrs[1];
2850
2851         if (!paddr || !qaddr)
2852                 addr = paddr ? paddr : qaddr;
2853
2854         switch (chan->device->id) {
2855         case PPC440SPE_DMA0_ID:
2856         case PPC440SPE_DMA1_ID:
2857                 /* walk through the WXOR source list and set P/Q-destinations
2858                  * for each slot:
2859                  */
2860                 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2861                         /* This is WXOR-only chain; may have 1/2 zero descs */
2862                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2863                                 index++;
2864                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2865                                 index++;
2866
2867                         iter = ppc440spe_get_group_entry(sw_desc, index);
2868                         if (addr) {
2869                                 /* one destination */
2870                                 list_for_each_entry_from(iter,
2871                                         &sw_desc->group_list, chain_node)
2872                                         ppc440spe_desc_set_dest_addr(iter, chan,
2873                                                 DMA_CUED_XOR_BASE, addr, 0);
2874                         } else {
2875                                 /* two destinations */
2876                                 list_for_each_entry_from(iter,
2877                                         &sw_desc->group_list, chain_node) {
2878                                         ppc440spe_desc_set_dest_addr(iter, chan,
2879                                                 DMA_CUED_XOR_BASE, paddr, 0);
2880                                         ppc440spe_desc_set_dest_addr(iter, chan,
2881                                                 DMA_CUED_XOR_BASE, qaddr, 1);
2882                                 }
2883                         }
2884
2885                         if (index) {
2886                                 /*  To clear destinations update the descriptor
2887                                  * (1st,2nd, or both depending on flags)
2888                                  */
2889                                 index = 0;
2890                                 if (test_bit(PPC440SPE_ZERO_P,
2891                                                 &sw_desc->flags)) {
2892                                         iter = ppc440spe_get_group_entry(
2893                                                         sw_desc, index++);
2894                                         ppc440spe_adma_pq_zero_op(iter, chan,
2895                                                         paddr);
2896                                 }
2897
2898                                 if (test_bit(PPC440SPE_ZERO_Q,
2899                                                 &sw_desc->flags)) {
2900                                         iter = ppc440spe_get_group_entry(
2901                                                         sw_desc, index++);
2902                                         ppc440spe_adma_pq_zero_op(iter, chan,
2903                                                         qaddr);
2904                                 }
2905
2906                                 return;
2907                         }
2908                 } else {
2909                         /* This is RXOR-only or RXOR/WXOR mixed chain */
2910
2911                         /* If we want to include destination into calculations,
2912                          * then make dest addresses cued with mult=1 (XOR).
2913                          */
2914                         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2915                                         DMA_CUED_XOR_HB :
2916                                         DMA_CUED_XOR_BASE |
2917                                                 (1 << DMA_CUED_MULT1_OFF);
2918                         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2919                                         DMA_CUED_XOR_HB :
2920                                         DMA_CUED_XOR_BASE |
2921                                                 (1 << DMA_CUED_MULT1_OFF);
2922
2923                         /* Setup destination(s) in RXOR slot(s) */
2924                         iter = ppc440spe_get_group_entry(sw_desc, index++);
2925                         ppc440spe_desc_set_dest_addr(iter, chan,
2926                                                 paddr ? ppath : qpath,
2927                                                 paddr ? paddr : qaddr, 0);
2928                         if (!addr) {
2929                                 /* two destinations */
2930                                 iter = ppc440spe_get_group_entry(sw_desc,
2931                                                                  index++);
2932                                 ppc440spe_desc_set_dest_addr(iter, chan,
2933                                                 qpath, qaddr, 0);
2934                         }
2935
2936                         if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2937                                 /* Setup destination(s) in remaining WXOR
2938                                  * slots
2939                                  */
2940                                 iter = ppc440spe_get_group_entry(sw_desc,
2941                                                                  index);
2942                                 if (addr) {
2943                                         /* one destination */
2944                                         list_for_each_entry_from(iter,
2945                                             &sw_desc->group_list,
2946                                             chain_node)
2947                                                 ppc440spe_desc_set_dest_addr(
2948                                                         iter, chan,
2949                                                         DMA_CUED_XOR_BASE,
2950                                                         addr, 0);
2951
2952                                 } else {
2953                                         /* two destinations */
2954                                         list_for_each_entry_from(iter,
2955                                             &sw_desc->group_list,
2956                                             chain_node) {
2957                                                 ppc440spe_desc_set_dest_addr(
2958                                                         iter, chan,
2959                                                         DMA_CUED_XOR_BASE,
2960                                                         paddr, 0);
2961                                                 ppc440spe_desc_set_dest_addr(
2962                                                         iter, chan,
2963                                                         DMA_CUED_XOR_BASE,
2964                                                         qaddr, 1);
2965                                         }
2966                                 }
2967                         }
2968
2969                 }
2970                 break;
2971
2972         case PPC440SPE_XOR_ID:
2973                 /* DMA2 descriptors have only 1 destination, so there are
2974                  * two chains - one for each dest.
2975                  * If we want to include destination into calculations,
2976                  * then make dest addresses cued with mult=1 (XOR).
2977                  */
2978                 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2979                                 DMA_CUED_XOR_HB :
2980                                 DMA_CUED_XOR_BASE |
2981                                         (1 << DMA_CUED_MULT1_OFF);
2982
2983                 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2984                                 DMA_CUED_XOR_HB :
2985                                 DMA_CUED_XOR_BASE |
2986                                         (1 << DMA_CUED_MULT1_OFF);
2987
2988                 iter = ppc440spe_get_group_entry(sw_desc, 0);
2989                 for (i = 0; i < sw_desc->descs_per_op; i++) {
2990                         ppc440spe_desc_set_dest_addr(iter, chan,
2991                                 paddr ? ppath : qpath,
2992                                 paddr ? paddr : qaddr, 0);
2993                         iter = list_entry(iter->chain_node.next,
2994                                           struct ppc440spe_adma_desc_slot,
2995                                           chain_node);
2996                 }
2997
2998                 if (!addr) {
2999                         /* Two destinations; setup Q here */
3000                         iter = ppc440spe_get_group_entry(sw_desc,
3001                                 sw_desc->descs_per_op);
3002                         for (i = 0; i < sw_desc->descs_per_op; i++) {
3003                                 ppc440spe_desc_set_dest_addr(iter,
3004                                         chan, qpath, qaddr, 0);
3005                                 iter = list_entry(iter->chain_node.next,
3006                                                 struct ppc440spe_adma_desc_slot,
3007                                                 chain_node);
3008                         }
3009                 }
3010
3011                 break;
3012         }
3013 }
3014
3015 /**
3016  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3017  * for the PQ_ZERO_SUM operation
3018  */
3019 static void ppc440spe_adma_pqzero_sum_set_dest(
3020                 struct ppc440spe_adma_desc_slot *sw_desc,
3021                 dma_addr_t paddr, dma_addr_t qaddr)
3022 {
3023         struct ppc440spe_adma_desc_slot *iter, *end;
3024         struct ppc440spe_adma_chan *chan;
3025         dma_addr_t addr = 0;
3026         int idx;
3027
3028         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3029
3030         /* walk through the WXOR source list and set P/Q-destinations
3031          * for each slot
3032          */
3033         idx = (paddr && qaddr) ? 2 : 1;
3034         /* set end */
3035         list_for_each_entry_reverse(end, &sw_desc->group_list,
3036                                     chain_node) {
3037                 if (!(--idx))
3038                         break;
3039         }
3040         /* set start */
3041         idx = (paddr && qaddr) ? 2 : 1;
3042         iter = ppc440spe_get_group_entry(sw_desc, idx);
3043
3044         if (paddr && qaddr) {
3045                 /* two destinations */
3046                 list_for_each_entry_from(iter, &sw_desc->group_list,
3047                                          chain_node) {
3048                         if (unlikely(iter == end))
3049                                 break;
3050                         ppc440spe_desc_set_dest_addr(iter, chan,
3051                                                 DMA_CUED_XOR_BASE, paddr, 0);
3052                         ppc440spe_desc_set_dest_addr(iter, chan,
3053                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3054                 }
3055         } else {
3056                 /* one destination */
3057                 addr = paddr ? paddr : qaddr;
3058                 list_for_each_entry_from(iter, &sw_desc->group_list,
3059                                          chain_node) {
3060                         if (unlikely(iter == end))
3061                                 break;
3062                         ppc440spe_desc_set_dest_addr(iter, chan,
3063                                                 DMA_CUED_XOR_BASE, addr, 0);
3064                 }
3065         }
3066
3067         /*  The remaining descriptors are DATACHECK. These have no need in
3068          * destination. Actually, these destinations are used there
3069          * as sources for check operation. So, set addr as source.
3070          */
3071         ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3072
3073         if (!addr) {
3074                 end = list_entry(end->chain_node.next,
3075                                  struct ppc440spe_adma_desc_slot, chain_node);
3076                 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3077         }
3078 }
3079
3080 /**
3081  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3082  */
3083 static inline void ppc440spe_desc_set_xor_src_cnt(
3084                         struct ppc440spe_adma_desc_slot *desc,
3085                         int src_cnt)
3086 {
3087         struct xor_cb *hw_desc = desc->hw_desc;
3088
3089         hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3090         hw_desc->cbc |= src_cnt;
3091 }
3092
3093 /**
3094  * ppc440spe_adma_pq_set_src - set source address into descriptor
3095  */
3096 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3097                 dma_addr_t addr, int index)
3098 {
3099         struct ppc440spe_adma_chan *chan;
3100         dma_addr_t haddr = 0;
3101         struct ppc440spe_adma_desc_slot *iter = NULL;
3102
3103         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3104
3105         switch (chan->device->id) {
3106         case PPC440SPE_DMA0_ID:
3107         case PPC440SPE_DMA1_ID:
3108                 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3109                  */
3110                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3111                         /* RXOR-only or RXOR/WXOR operation */
3112                         int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3113                                 &sw_desc->flags) ?  2 : 3;
3114
3115                         if (index == 0) {
3116                                 /* 1st slot (RXOR) */
3117                                 /* setup sources region (R1-2-3, R1-2-4,
3118                                  * or R1-2-5)
3119                                  */
3120                                 if (test_bit(PPC440SPE_DESC_RXOR12,
3121                                                 &sw_desc->flags))
3122                                         haddr = DMA_RXOR12 <<
3123                                                 DMA_CUED_REGION_OFF;
3124                                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3125                                     &sw_desc->flags))
3126                                         haddr = DMA_RXOR123 <<
3127                                                 DMA_CUED_REGION_OFF;
3128                                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3129                                     &sw_desc->flags))
3130                                         haddr = DMA_RXOR124 <<
3131                                                 DMA_CUED_REGION_OFF;
3132                                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3133                                     &sw_desc->flags))
3134                                         haddr = DMA_RXOR125 <<
3135                                                 DMA_CUED_REGION_OFF;
3136                                 else
3137                                         BUG();
3138                                 haddr |= DMA_CUED_XOR_BASE;
3139                                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3140                         } else if (index < iskip) {
3141                                 /* 1st slot (RXOR)
3142                                  * shall actually set source address only once
3143                                  * instead of first <iskip>
3144                                  */
3145                                 iter = NULL;
3146                         } else {
3147                                 /* 2nd/3d and next slots (WXOR);
3148                                  * skip first slot with RXOR
3149                                  */
3150                                 haddr = DMA_CUED_XOR_HB;
3151                                 iter = ppc440spe_get_group_entry(sw_desc,
3152                                     index - iskip + sw_desc->dst_cnt);
3153                         }
3154                 } else {
3155                         int znum = 0;
3156
3157                         /* WXOR-only operation; skip first slots with
3158                          * zeroing destinations
3159                          */
3160                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3161                                 znum++;
3162                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3163                                 znum++;
3164
3165                         haddr = DMA_CUED_XOR_HB;
3166                         iter = ppc440spe_get_group_entry(sw_desc,
3167                                         index + znum);
3168                 }
3169
3170                 if (likely(iter)) {
3171                         ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3172
3173                         if (!index &&
3174                             test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3175                             sw_desc->dst_cnt == 2) {
3176                                 /* if we have two destinations for RXOR, then
3177                                  * setup source in the second descr too
3178                                  */
3179                                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3180                                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3181                                         haddr, addr);
3182                         }
3183                 }
3184                 break;
3185
3186         case PPC440SPE_XOR_ID:
3187                 /* DMA2 may do Biskup */
3188                 iter = sw_desc->group_head;
3189                 if (iter->dst_cnt == 2) {
3190                         /* both P & Q calculations required; set P src here */
3191                         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3192
3193                         /* this is for Q */
3194                         iter = ppc440spe_get_group_entry(sw_desc,
3195                                 sw_desc->descs_per_op);
3196                 }
3197                 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3198                 break;
3199         }
3200 }
3201
3202 /**
3203  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3204  */
3205 static void ppc440spe_adma_memcpy_xor_set_src(
3206                 struct ppc440spe_adma_desc_slot *sw_desc,
3207                 dma_addr_t addr, int index)
3208 {
3209         struct ppc440spe_adma_chan *chan;
3210
3211         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3212         sw_desc = sw_desc->group_head;
3213
3214         if (likely(sw_desc))
3215                 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3216 }
3217
3218 /**
3219  * ppc440spe_adma_dma2rxor_inc_addr  -
3220  */
3221 static void ppc440spe_adma_dma2rxor_inc_addr(
3222                 struct ppc440spe_adma_desc_slot *desc,
3223                 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3224 {
3225         cursor->addr_count++;
3226         if (index == src_cnt - 1) {
3227                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3228         } else if (cursor->addr_count == XOR_MAX_OPS) {
3229                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3230                 cursor->addr_count = 0;
3231                 cursor->desc_count++;
3232         }
3233 }
3234
3235 /**
3236  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3237  */
3238 static int ppc440spe_adma_dma2rxor_prep_src(
3239                 struct ppc440spe_adma_desc_slot *hdesc,
3240                 struct ppc440spe_rxor *cursor, int index,
3241                 int src_cnt, u32 addr)
3242 {
3243         int rval = 0;
3244         u32 sign;
3245         struct ppc440spe_adma_desc_slot *desc = hdesc;
3246         int i;
3247
3248         for (i = 0; i < cursor->desc_count; i++) {
3249                 desc = list_entry(hdesc->chain_node.next,
3250                                   struct ppc440spe_adma_desc_slot,
3251                                   chain_node);
3252         }
3253
3254         switch (cursor->state) {
3255         case 0:
3256                 if (addr == cursor->addrl + cursor->len) {
3257                         /* direct RXOR */
3258                         cursor->state = 1;
3259                         cursor->xor_count++;
3260                         if (index == src_cnt-1) {
3261                                 ppc440spe_rxor_set_region(desc,
3262                                         cursor->addr_count,
3263                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3264                                 ppc440spe_adma_dma2rxor_inc_addr(
3265                                         desc, cursor, index, src_cnt);
3266                         }
3267                 } else if (cursor->addrl == addr + cursor->len) {
3268                         /* reverse RXOR */
3269                         cursor->state = 1;
3270                         cursor->xor_count++;
3271                         set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3272                         if (index == src_cnt-1) {
3273                                 ppc440spe_rxor_set_region(desc,
3274                                         cursor->addr_count,
3275                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3276                                 ppc440spe_adma_dma2rxor_inc_addr(
3277                                         desc, cursor, index, src_cnt);
3278                         }
3279                 } else {
3280                         printk(KERN_ERR "Cannot build "
3281                                 "DMA2 RXOR command block.\n");
3282                         BUG();
3283                 }
3284                 break;
3285         case 1:
3286                 sign = test_bit(cursor->addr_count,
3287                                 desc->reverse_flags)
3288                         ? -1 : 1;
3289                 if (index == src_cnt-2 || (sign == -1
3290                         && addr != cursor->addrl - 2*cursor->len)) {
3291                         cursor->state = 0;
3292                         cursor->xor_count = 1;
3293                         cursor->addrl = addr;
3294                         ppc440spe_rxor_set_region(desc,
3295                                 cursor->addr_count,
3296                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3297                         ppc440spe_adma_dma2rxor_inc_addr(
3298                                 desc, cursor, index, src_cnt);
3299                 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3300                         cursor->state = 2;
3301                         cursor->xor_count = 0;
3302                         ppc440spe_rxor_set_region(desc,
3303                                 cursor->addr_count,
3304                                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3305                         if (index == src_cnt-1) {
3306                                 ppc440spe_adma_dma2rxor_inc_addr(
3307                                         desc, cursor, index, src_cnt);
3308                         }
3309                 } else if (addr == cursor->addrl + 3*cursor->len) {
3310                         cursor->state = 2;
3311                         cursor->xor_count = 0;
3312                         ppc440spe_rxor_set_region(desc,
3313                                 cursor->addr_count,
3314                                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3315                         if (index == src_cnt-1) {
3316                                 ppc440spe_adma_dma2rxor_inc_addr(
3317                                         desc, cursor, index, src_cnt);
3318                         }
3319                 } else if (addr == cursor->addrl + 4*cursor->len) {
3320                         cursor->state = 2;
3321                         cursor->xor_count = 0;
3322                         ppc440spe_rxor_set_region(desc,
3323                                 cursor->addr_count,
3324                                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3325                         if (index == src_cnt-1) {
3326                                 ppc440spe_adma_dma2rxor_inc_addr(
3327                                         desc, cursor, index, src_cnt);
3328                         }
3329                 } else {
3330                         cursor->state = 0;
3331                         cursor->xor_count = 1;
3332                         cursor->addrl = addr;
3333                         ppc440spe_rxor_set_region(desc,
3334                                 cursor->addr_count,
3335                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3336                         ppc440spe_adma_dma2rxor_inc_addr(
3337                                 desc, cursor, index, src_cnt);
3338                 }
3339                 break;
3340         case 2:
3341                 cursor->state = 0;
3342                 cursor->addrl = addr;
3343                 cursor->xor_count++;
3344                 if (index) {
3345                         ppc440spe_adma_dma2rxor_inc_addr(
3346                                 desc, cursor, index, src_cnt);
3347                 }
3348                 break;
3349         }
3350
3351         return rval;
3352 }
3353
3354 /**
3355  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3356  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3357  */
3358 static void ppc440spe_adma_dma2rxor_set_src(
3359                 struct ppc440spe_adma_desc_slot *desc,
3360                 int index, dma_addr_t addr)
3361 {
3362         struct xor_cb *xcb = desc->hw_desc;
3363         int k = 0, op = 0, lop = 0;
3364
3365         /* get the RXOR operand which corresponds to index addr */
3366         while (op <= index) {
3367                 lop = op;
3368                 if (k == XOR_MAX_OPS) {
3369                         k = 0;
3370                         desc = list_entry(desc->chain_node.next,
3371                                 struct ppc440spe_adma_desc_slot, chain_node);
3372                         xcb = desc->hw_desc;
3373
3374                 }
3375                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3376                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3377                         op += 2;
3378                 else
3379                         op += 3;
3380         }
3381
3382         BUG_ON(k < 1);
3383
3384         if (test_bit(k-1, desc->reverse_flags)) {
3385                 /* reverse operand order; put last op in RXOR group */
3386                 if (index == op - 1)
3387                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3388         } else {
3389                 /* direct operand order; put first op in RXOR group */
3390                 if (index == lop)
3391                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3392         }
3393 }
3394
3395 /**
3396  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3397  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3398  */
3399 static void ppc440spe_adma_dma2rxor_set_mult(
3400                 struct ppc440spe_adma_desc_slot *desc,
3401                 int index, u8 mult)
3402 {
3403         struct xor_cb *xcb = desc->hw_desc;
3404         int k = 0, op = 0, lop = 0;
3405
3406         /* get the RXOR operand which corresponds to index mult */
3407         while (op <= index) {
3408                 lop = op;
3409                 if (k == XOR_MAX_OPS) {
3410                         k = 0;
3411                         desc = list_entry(desc->chain_node.next,
3412                                           struct ppc440spe_adma_desc_slot,
3413                                           chain_node);
3414                         xcb = desc->hw_desc;
3415
3416                 }
3417                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3418                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3419                         op += 2;
3420                 else
3421                         op += 3;
3422         }
3423
3424         BUG_ON(k < 1);
3425         if (test_bit(k-1, desc->reverse_flags)) {
3426                 /* reverse order */
3427                 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3428         } else {
3429                 /* direct order */
3430                 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3431         }
3432 }
3433
3434 /**
3435  * ppc440spe_init_rxor_cursor -
3436  */
3437 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3438 {
3439         memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3440         cursor->state = 2;
3441 }
3442
3443 /**
3444  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3445  * descriptor for the PQXOR operation
3446  */
3447 static void ppc440spe_adma_pq_set_src_mult(
3448                 struct ppc440spe_adma_desc_slot *sw_desc,
3449                 unsigned char mult, int index, int dst_pos)
3450 {
3451         struct ppc440spe_adma_chan *chan;
3452         u32 mult_idx, mult_dst;
3453         struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3454
3455         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3456
3457         switch (chan->device->id) {
3458         case PPC440SPE_DMA0_ID:
3459         case PPC440SPE_DMA1_ID:
3460                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3461                         int region = test_bit(PPC440SPE_DESC_RXOR12,
3462                                         &sw_desc->flags) ? 2 : 3;
3463
3464                         if (index < region) {
3465                                 /* RXOR multipliers */
3466                                 iter = ppc440spe_get_group_entry(sw_desc,
3467                                         sw_desc->dst_cnt - 1);
3468                                 if (sw_desc->dst_cnt == 2)
3469                                         iter1 = ppc440spe_get_group_entry(
3470                                                         sw_desc, 0);
3471
3472                                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3473                                 mult_dst = DMA_CDB_SG_SRC;
3474                         } else {
3475                                 /* WXOR multiplier */
3476                                 iter = ppc440spe_get_group_entry(sw_desc,
3477                                                         index - region +
3478                                                         sw_desc->dst_cnt);
3479                                 mult_idx = DMA_CUED_MULT1_OFF;
3480                                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3481                                                      DMA_CDB_SG_DST1;
3482                         }
3483                 } else {
3484                         int znum = 0;
3485
3486                         /* WXOR-only;
3487                          * skip first slots with destinations (if ZERO_DST has
3488                          * place)
3489                          */
3490                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3491                                 znum++;
3492                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3493                                 znum++;
3494
3495                         iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3496                         mult_idx = DMA_CUED_MULT1_OFF;
3497                         mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3498                 }
3499
3500                 if (likely(iter)) {
3501                         ppc440spe_desc_set_src_mult(iter, chan,
3502                                 mult_idx, mult_dst, mult);
3503
3504                         if (unlikely(iter1)) {
3505                                 /* if we have two destinations for RXOR, then
3506                                  * we've just set Q mult. Set-up P now.
3507                                  */
3508                                 ppc440spe_desc_set_src_mult(iter1, chan,
3509                                         mult_idx, mult_dst, 1);
3510                         }
3511
3512                 }
3513                 break;
3514
3515         case PPC440SPE_XOR_ID:
3516                 iter = sw_desc->group_head;
3517                 if (sw_desc->dst_cnt == 2) {
3518                         /* both P & Q calculations required; set P mult here */
3519                         ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3520
3521                         /* and then set Q mult */
3522                         iter = ppc440spe_get_group_entry(sw_desc,
3523                                sw_desc->descs_per_op);
3524                 }
3525                 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3526                 break;
3527         }
3528 }
3529
3530 /**
3531  * ppc440spe_adma_free_chan_resources - free the resources allocated
3532  */
3533 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3534 {
3535         struct ppc440spe_adma_chan *ppc440spe_chan;
3536         struct ppc440spe_adma_desc_slot *iter, *_iter;
3537         int in_use_descs = 0;
3538
3539         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3540         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3541
3542         spin_lock_bh(&ppc440spe_chan->lock);
3543         list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3544                                         chain_node) {
3545                 in_use_descs++;
3546                 list_del(&iter->chain_node);
3547         }
3548         list_for_each_entry_safe_reverse(iter, _iter,
3549                         &ppc440spe_chan->all_slots, slot_node) {
3550                 list_del(&iter->slot_node);
3551                 kfree(iter);
3552                 ppc440spe_chan->slots_allocated--;
3553         }
3554         ppc440spe_chan->last_used = NULL;
3555
3556         dev_dbg(ppc440spe_chan->device->common.dev,
3557                 "ppc440spe adma%d %s slots_allocated %d\n",
3558                 ppc440spe_chan->device->id,
3559                 __func__, ppc440spe_chan->slots_allocated);
3560         spin_unlock_bh(&ppc440spe_chan->lock);
3561
3562         /* one is ok since we left it on there on purpose */
3563         if (in_use_descs > 1)
3564                 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3565                         in_use_descs - 1);
3566 }
3567
3568 /**
3569  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3570  * @chan: ADMA channel handle
3571  * @cookie: ADMA transaction identifier
3572  * @txstate: a holder for the current state of the channel
3573  */
3574 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3575                         dma_cookie_t cookie, struct dma_tx_state *txstate)
3576 {
3577         struct ppc440spe_adma_chan *ppc440spe_chan;
3578         enum dma_status ret;
3579
3580         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3581         ret = dma_cookie_status(chan, cookie, txstate);
3582         if (ret == DMA_COMPLETE)
3583                 return ret;
3584
3585         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3586
3587         return dma_cookie_status(chan, cookie, txstate);
3588 }
3589
3590 /**
3591  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3592  */
3593 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3594 {
3595         struct ppc440spe_adma_chan *chan = data;
3596
3597         dev_dbg(chan->device->common.dev,
3598                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3599
3600         tasklet_schedule(&chan->irq_tasklet);
3601         ppc440spe_adma_device_clear_eot_status(chan);
3602
3603         return IRQ_HANDLED;
3604 }
3605
3606 /**
3607  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3608  *      do the same things as a eot handler
3609  */
3610 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3611 {
3612         struct ppc440spe_adma_chan *chan = data;
3613
3614         dev_dbg(chan->device->common.dev,
3615                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3616
3617         tasklet_schedule(&chan->irq_tasklet);
3618         ppc440spe_adma_device_clear_eot_status(chan);
3619
3620         return IRQ_HANDLED;
3621 }
3622
3623 /**
3624  * ppc440spe_test_callback - called when test operation has been done
3625  */
3626 static void ppc440spe_test_callback(void *unused)
3627 {
3628         complete(&ppc440spe_r6_test_comp);
3629 }
3630
3631 /**
3632  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3633  */
3634 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3635 {
3636         struct ppc440spe_adma_chan *ppc440spe_chan;
3637
3638         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3639         dev_dbg(ppc440spe_chan->device->common.dev,
3640                 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3641                 __func__, ppc440spe_chan->pending);
3642
3643         if (ppc440spe_chan->pending) {
3644                 ppc440spe_chan->pending = 0;
3645                 ppc440spe_chan_append(ppc440spe_chan);
3646         }
3647 }
3648
3649 /**
3650  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3651  *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3652  *      specific operation)
3653  */
3654 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3655 {
3656         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3657         dma_cookie_t cookie;
3658         int slot_cnt, slots_per_op;
3659
3660         dev_dbg(chan->device->common.dev,
3661                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3662
3663         spin_lock_bh(&chan->lock);
3664         slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3665         sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3666         if (sw_desc) {
3667                 group_start = sw_desc->group_head;
3668                 list_splice_init(&sw_desc->group_list, &chan->chain);
3669                 async_tx_ack(&sw_desc->async_tx);
3670                 ppc440spe_desc_init_null_xor(group_start);
3671
3672                 cookie = dma_cookie_assign(&sw_desc->async_tx);
3673
3674                 /* initialize the completed cookie to be less than
3675                  * the most recently used cookie
3676                  */
3677                 chan->common.completed_cookie = cookie - 1;
3678
3679                 /* channel should not be busy */
3680                 BUG_ON(ppc440spe_chan_is_busy(chan));
3681
3682                 /* set the descriptor address */
3683                 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3684
3685                 /* run the descriptor */
3686                 ppc440spe_chan_run(chan);
3687         } else
3688                 printk(KERN_ERR "ppc440spe adma%d"
3689                         " failed to allocate null descriptor\n",
3690                         chan->device->id);
3691         spin_unlock_bh(&chan->lock);
3692 }
3693
3694 /**
3695  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3696  *      For this we just perform one WXOR operation with the same source
3697  *      and destination addresses, the GF-multiplier is 1; so if RAID-6
3698  *      capabilities are enabled then we'll get src/dst filled with zero.
3699  */
3700 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3701 {
3702         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3703         struct page *pg;
3704         char *a;
3705         dma_addr_t dma_addr, addrs[2];
3706         unsigned long op = 0;
3707         int rval = 0;
3708
3709         set_bit(PPC440SPE_DESC_WXOR, &op);
3710
3711         pg = alloc_page(GFP_KERNEL);
3712         if (!pg)
3713                 return -ENOMEM;
3714
3715         spin_lock_bh(&chan->lock);
3716         sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3717         if (sw_desc) {
3718                 /* 1 src, 1 dsr, int_ena, WXOR */
3719                 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3720                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3721                         ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3722                         iter->unmap_len = PAGE_SIZE;
3723                 }
3724         } else {
3725                 rval = -EFAULT;
3726                 spin_unlock_bh(&chan->lock);
3727                 goto exit;
3728         }
3729         spin_unlock_bh(&chan->lock);
3730
3731         /* Fill the test page with ones */
3732         memset(page_address(pg), 0xFF, PAGE_SIZE);
3733         dma_addr = dma_map_page(chan->device->dev, pg, 0,
3734                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3735
3736         /* Setup addresses */
3737         ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3738         ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3739         addrs[0] = dma_addr;
3740         addrs[1] = 0;
3741         ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3742
3743         async_tx_ack(&sw_desc->async_tx);
3744         sw_desc->async_tx.callback = ppc440spe_test_callback;
3745         sw_desc->async_tx.callback_param = NULL;
3746
3747         init_completion(&ppc440spe_r6_test_comp);
3748
3749         ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3750         ppc440spe_adma_issue_pending(&chan->common);
3751
3752         wait_for_completion(&ppc440spe_r6_test_comp);
3753
3754         /* Now check if the test page is zeroed */
3755         a = page_address(pg);
3756         if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3757                 /* page is zero - RAID-6 enabled */
3758                 rval = 0;
3759         } else {
3760                 /* RAID-6 was not enabled */
3761                 rval = -EINVAL;
3762         }
3763 exit:
3764         __free_page(pg);
3765         return rval;
3766 }
3767
3768 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3769 {
3770         switch (adev->id) {
3771         case PPC440SPE_DMA0_ID:
3772         case PPC440SPE_DMA1_ID:
3773                 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3774                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3775                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3776                 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3777                 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3778                 break;
3779         case PPC440SPE_XOR_ID:
3780                 dma_cap_set(DMA_XOR, adev->common.cap_mask);
3781                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3782                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3783                 adev->common.cap_mask = adev->common.cap_mask;
3784                 break;
3785         }
3786
3787         /* Set base routines */
3788         adev->common.device_alloc_chan_resources =
3789                                 ppc440spe_adma_alloc_chan_resources;
3790         adev->common.device_free_chan_resources =
3791                                 ppc440spe_adma_free_chan_resources;
3792         adev->common.device_tx_status = ppc440spe_adma_tx_status;
3793         adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3794
3795         /* Set prep routines based on capability */
3796         if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3797                 adev->common.device_prep_dma_memcpy =
3798                         ppc440spe_adma_prep_dma_memcpy;
3799         }
3800         if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3801                 adev->common.max_xor = XOR_MAX_OPS;
3802                 adev->common.device_prep_dma_xor =
3803                         ppc440spe_adma_prep_dma_xor;
3804         }
3805         if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3806                 switch (adev->id) {
3807                 case PPC440SPE_DMA0_ID:
3808                         dma_set_maxpq(&adev->common,
3809                                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3810                         break;
3811                 case PPC440SPE_DMA1_ID:
3812                         dma_set_maxpq(&adev->common,
3813                                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3814                         break;
3815                 case PPC440SPE_XOR_ID:
3816                         adev->common.max_pq = XOR_MAX_OPS * 3;
3817                         break;
3818                 }
3819                 adev->common.device_prep_dma_pq =
3820                         ppc440spe_adma_prep_dma_pq;
3821         }
3822         if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3823                 switch (adev->id) {
3824                 case PPC440SPE_DMA0_ID:
3825                         adev->common.max_pq = DMA0_FIFO_SIZE /
3826                                                 sizeof(struct dma_cdb);
3827                         break;
3828                 case PPC440SPE_DMA1_ID:
3829                         adev->common.max_pq = DMA1_FIFO_SIZE /
3830                                                 sizeof(struct dma_cdb);
3831                         break;
3832                 }
3833                 adev->common.device_prep_dma_pq_val =
3834                         ppc440spe_adma_prep_dma_pqzero_sum;
3835         }
3836         if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3837                 switch (adev->id) {
3838                 case PPC440SPE_DMA0_ID:
3839                         adev->common.max_xor = DMA0_FIFO_SIZE /
3840                                                 sizeof(struct dma_cdb);
3841                         break;
3842                 case PPC440SPE_DMA1_ID:
3843                         adev->common.max_xor = DMA1_FIFO_SIZE /
3844                                                 sizeof(struct dma_cdb);
3845                         break;
3846                 }
3847                 adev->common.device_prep_dma_xor_val =
3848                         ppc440spe_adma_prep_dma_xor_zero_sum;
3849         }
3850         if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3851                 adev->common.device_prep_dma_interrupt =
3852                         ppc440spe_adma_prep_dma_interrupt;
3853         }
3854         pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3855           "( %s%s%s%s%s%s)\n",
3856           dev_name(adev->dev),
3857           dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3858           dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3859           dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3860           dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3861           dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3862           dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3863 }
3864
3865 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3866                                      struct ppc440spe_adma_chan *chan,
3867                                      int *initcode)
3868 {
3869         struct platform_device *ofdev;
3870         struct device_node *np;
3871         int ret;
3872
3873         ofdev = container_of(adev->dev, struct platform_device, dev);
3874         np = ofdev->dev.of_node;
3875         if (adev->id != PPC440SPE_XOR_ID) {
3876                 adev->err_irq = irq_of_parse_and_map(np, 1);
3877                 if (!adev->err_irq) {
3878                         dev_warn(adev->dev, "no err irq resource?\n");
3879                         *initcode = PPC_ADMA_INIT_IRQ2;
3880                         adev->err_irq = -ENXIO;
3881                 } else
3882                         atomic_inc(&ppc440spe_adma_err_irq_ref);
3883         } else {
3884                 adev->err_irq = -ENXIO;
3885         }
3886
3887         adev->irq = irq_of_parse_and_map(np, 0);
3888         if (!adev->irq) {
3889                 dev_err(adev->dev, "no irq resource\n");
3890                 *initcode = PPC_ADMA_INIT_IRQ1;
3891                 ret = -ENXIO;
3892                 goto err_irq_map;
3893         }
3894         dev_dbg(adev->dev, "irq %d, err irq %d\n",
3895                 adev->irq, adev->err_irq);
3896
3897         ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3898                           0, dev_driver_string(adev->dev), chan);
3899         if (ret) {
3900                 dev_err(adev->dev, "can't request irq %d\n",
3901                         adev->irq);
3902                 *initcode = PPC_ADMA_INIT_IRQ1;
3903                 ret = -EIO;
3904                 goto err_req1;
3905         }
3906
3907         /* only DMA engines have a separate error IRQ
3908          * so it's Ok if err_irq < 0 in XOR engine case.
3909          */
3910         if (adev->err_irq > 0) {
3911                 /* both DMA engines share common error IRQ */
3912                 ret = request_irq(adev->err_irq,
3913                                   ppc440spe_adma_err_handler,
3914                                   IRQF_SHARED,
3915                                   dev_driver_string(adev->dev),
3916                                   chan);
3917                 if (ret) {
3918                         dev_err(adev->dev, "can't request irq %d\n",
3919                                 adev->err_irq);
3920                         *initcode = PPC_ADMA_INIT_IRQ2;
3921                         ret = -EIO;
3922                         goto err_req2;
3923                 }
3924         }
3925
3926         if (adev->id == PPC440SPE_XOR_ID) {
3927                 /* enable XOR engine interrupts */
3928                 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3929                             XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3930                             &adev->xor_reg->ier);
3931         } else {
3932                 u32 mask, enable;
3933
3934                 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3935                 if (!np) {
3936                         pr_err("%s: can't find I2O device tree node\n",
3937                                 __func__);
3938                         ret = -ENODEV;
3939                         goto err_req2;
3940                 }
3941                 adev->i2o_reg = of_iomap(np, 0);
3942                 if (!adev->i2o_reg) {
3943                         pr_err("%s: failed to map I2O registers\n", __func__);
3944                         of_node_put(np);
3945                         ret = -EINVAL;
3946                         goto err_req2;
3947                 }
3948                 of_node_put(np);
3949                 /* Unmask 'CS FIFO Attention' interrupts and
3950                  * enable generating interrupts on errors
3951                  */
3952                 enable = (adev->id == PPC440SPE_DMA0_ID) ?
3953                          ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3954                          ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3955                 mask = ioread32(&adev->i2o_reg->iopim) & enable;
3956                 iowrite32(mask, &adev->i2o_reg->iopim);
3957         }
3958         return 0;
3959
3960 err_req2:
3961         free_irq(adev->irq, chan);
3962 err_req1:
3963         irq_dispose_mapping(adev->irq);
3964 err_irq_map:
3965         if (adev->err_irq > 0) {
3966                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3967                         irq_dispose_mapping(adev->err_irq);
3968         }
3969         return ret;
3970 }
3971
3972 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3973                                         struct ppc440spe_adma_chan *chan)
3974 {
3975         u32 mask, disable;
3976
3977         if (adev->id == PPC440SPE_XOR_ID) {
3978                 /* disable XOR engine interrupts */
3979                 mask = ioread32be(&adev->xor_reg->ier);
3980                 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3981                           XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3982                 iowrite32be(mask, &adev->xor_reg->ier);
3983         } else {
3984                 /* disable DMAx engine interrupts */
3985                 disable = (adev->id == PPC440SPE_DMA0_ID) ?
3986                           (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3987                           (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3988                 mask = ioread32(&adev->i2o_reg->iopim) | disable;
3989                 iowrite32(mask, &adev->i2o_reg->iopim);
3990         }
3991         free_irq(adev->irq, chan);
3992         irq_dispose_mapping(adev->irq);
3993         if (adev->err_irq > 0) {
3994                 free_irq(adev->err_irq, chan);
3995                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
3996                         irq_dispose_mapping(adev->err_irq);
3997                         iounmap(adev->i2o_reg);
3998                 }
3999         }
4000 }
4001
4002 /**
4003  * ppc440spe_adma_probe - probe the asynch device
4004  */
4005 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4006 {
4007         struct device_node *np = ofdev->dev.of_node;
4008         struct resource res;
4009         struct ppc440spe_adma_device *adev;
4010         struct ppc440spe_adma_chan *chan;
4011         struct ppc_dma_chan_ref *ref, *_ref;
4012         int ret = 0, initcode = PPC_ADMA_INIT_OK;
4013         const u32 *idx;
4014         int len;
4015         void *regs;
4016         u32 id, pool_size;
4017
4018         if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4019                 id = PPC440SPE_XOR_ID;
4020                 /* As far as the XOR engine is concerned, it does not
4021                  * use FIFOs but uses linked list. So there is no dependency
4022                  * between pool size to allocate and the engine configuration.
4023                  */
4024                 pool_size = PAGE_SIZE << 1;
4025         } else {
4026                 /* it is DMA0 or DMA1 */
4027                 idx = of_get_property(np, "cell-index", &len);
4028                 if (!idx || (len != sizeof(u32))) {
4029                         dev_err(&ofdev->dev, "Device node %pOF has missing "
4030                                 "or invalid cell-index property\n",
4031                                 np);
4032                         return -EINVAL;
4033                 }
4034                 id = *idx;
4035                 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4036                  * should allocate the pool accordingly to size of this
4037                  * FIFO. Thus, the pool size depends on the FIFO depth:
4038                  * how much CDBs pointers the FIFO may contain then so
4039                  * much CDBs we should provide in the pool.
4040                  * That is
4041                  *   CDB size = 32B;
4042                  *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4043                  *   Pool size = CDBs number * CDB size =
4044                  *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4045                  */
4046                 pool_size = (id == PPC440SPE_DMA0_ID) ?
4047                             DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4048                 pool_size <<= 2;
4049         }
4050
4051         if (of_address_to_resource(np, 0, &res)) {
4052                 dev_err(&ofdev->dev, "failed to get memory resource\n");
4053                 initcode = PPC_ADMA_INIT_MEMRES;
4054                 ret = -ENODEV;
4055                 goto out;
4056         }
4057
4058         if (!request_mem_region(res.start, resource_size(&res),
4059                                 dev_driver_string(&ofdev->dev))) {
4060                 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4061                         &res);
4062                 initcode = PPC_ADMA_INIT_MEMREG;
4063                 ret = -EBUSY;
4064                 goto out;
4065         }
4066
4067         /* create a device */
4068         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4069         if (!adev) {
4070                 initcode = PPC_ADMA_INIT_ALLOC;
4071                 ret = -ENOMEM;
4072                 goto err_adev_alloc;
4073         }
4074
4075         adev->id = id;
4076         adev->pool_size = pool_size;
4077         /* allocate coherent memory for hardware descriptors */
4078         adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4079                                         adev->pool_size, &adev->dma_desc_pool,
4080                                         GFP_KERNEL);
4081         if (adev->dma_desc_pool_virt == NULL) {
4082                 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4083                         "memory for hardware descriptors\n",
4084                         adev->pool_size);
4085                 initcode = PPC_ADMA_INIT_COHERENT;
4086                 ret = -ENOMEM;
4087                 goto err_dma_alloc;
4088         }
4089         dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4090                 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4091
4092         regs = ioremap(res.start, resource_size(&res));
4093         if (!regs) {
4094                 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4095                 ret = -ENOMEM;
4096                 goto err_regs_alloc;
4097         }
4098
4099         if (adev->id == PPC440SPE_XOR_ID) {
4100                 adev->xor_reg = regs;
4101                 /* Reset XOR */
4102                 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4103                 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4104         } else {
4105                 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4106                                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4107                 adev->dma_reg = regs;
4108                 /* DMAx_FIFO_SIZE is defined in bytes,
4109                  * <fsiz> - is defined in number of CDB pointers (8byte).
4110                  * DMA FIFO Length = CSlength + CPlength, where
4111                  * CSlength = CPlength = (fsiz + 1) * 8.
4112                  */
4113                 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4114                           &adev->dma_reg->fsiz);
4115                 /* Configure DMA engine */
4116                 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4117                           &adev->dma_reg->cfg);
4118                 /* Clear Status */
4119                 iowrite32(~0, &adev->dma_reg->dsts);
4120         }
4121
4122         adev->dev = &ofdev->dev;
4123         adev->common.dev = &ofdev->dev;
4124         INIT_LIST_HEAD(&adev->common.channels);
4125         platform_set_drvdata(ofdev, adev);
4126
4127         /* create a channel */
4128         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4129         if (!chan) {
4130                 initcode = PPC_ADMA_INIT_CHANNEL;
4131                 ret = -ENOMEM;
4132                 goto err_chan_alloc;
4133         }
4134
4135         spin_lock_init(&chan->lock);
4136         INIT_LIST_HEAD(&chan->chain);
4137         INIT_LIST_HEAD(&chan->all_slots);
4138         chan->device = adev;
4139         chan->common.device = &adev->common;
4140         dma_cookie_init(&chan->common);
4141         list_add_tail(&chan->common.device_node, &adev->common.channels);
4142         tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet);
4143
4144         /* allocate and map helper pages for async validation or
4145          * async_mult/async_sum_product operations on DMA0/1.
4146          */
4147         if (adev->id != PPC440SPE_XOR_ID) {
4148                 chan->pdest_page = alloc_page(GFP_KERNEL);
4149                 chan->qdest_page = alloc_page(GFP_KERNEL);
4150                 if (!chan->pdest_page ||
4151                     !chan->qdest_page) {
4152                         if (chan->pdest_page)
4153                                 __free_page(chan->pdest_page);
4154                         if (chan->qdest_page)
4155                                 __free_page(chan->qdest_page);
4156                         ret = -ENOMEM;
4157                         goto err_page_alloc;
4158                 }
4159                 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4160                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4161                 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4162                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4163         }
4164
4165         ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4166         if (ref) {
4167                 ref->chan = &chan->common;
4168                 INIT_LIST_HEAD(&ref->node);
4169                 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4170         } else {
4171                 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4172                 ret = -ENOMEM;
4173                 goto err_ref_alloc;
4174         }
4175
4176         ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4177         if (ret)
4178                 goto err_irq;
4179
4180         ppc440spe_adma_init_capabilities(adev);
4181
4182         ret = dma_async_device_register(&adev->common);
4183         if (ret) {
4184                 initcode = PPC_ADMA_INIT_REGISTER;
4185                 dev_err(&ofdev->dev, "failed to register dma device\n");
4186                 goto err_dev_reg;
4187         }
4188
4189         goto out;
4190
4191 err_dev_reg:
4192         ppc440spe_adma_release_irqs(adev, chan);
4193 err_irq:
4194         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4195                 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4196                         list_del(&ref->node);
4197                         kfree(ref);
4198                 }
4199         }
4200 err_ref_alloc:
4201         if (adev->id != PPC440SPE_XOR_ID) {
4202                 dma_unmap_page(&ofdev->dev, chan->pdest,
4203                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4204                 dma_unmap_page(&ofdev->dev, chan->qdest,
4205                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4206                 __free_page(chan->pdest_page);
4207                 __free_page(chan->qdest_page);
4208         }
4209 err_page_alloc:
4210         kfree(chan);
4211 err_chan_alloc:
4212         if (adev->id == PPC440SPE_XOR_ID)
4213                 iounmap(adev->xor_reg);
4214         else
4215                 iounmap(adev->dma_reg);
4216 err_regs_alloc:
4217         dma_free_coherent(adev->dev, adev->pool_size,
4218                           adev->dma_desc_pool_virt,
4219                           adev->dma_desc_pool);
4220 err_dma_alloc:
4221         kfree(adev);
4222 err_adev_alloc:
4223         release_mem_region(res.start, resource_size(&res));
4224 out:
4225         if (id < PPC440SPE_ADMA_ENGINES_NUM)
4226                 ppc440spe_adma_devices[id] = initcode;
4227
4228         return ret;
4229 }
4230
4231 /**
4232  * ppc440spe_adma_remove - remove the asynch device
4233  */
4234 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4235 {
4236         struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4237         struct device_node *np = ofdev->dev.of_node;
4238         struct resource res;
4239         struct dma_chan *chan, *_chan;
4240         struct ppc_dma_chan_ref *ref, *_ref;
4241         struct ppc440spe_adma_chan *ppc440spe_chan;
4242
4243         if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4244                 ppc440spe_adma_devices[adev->id] = -1;
4245
4246         dma_async_device_unregister(&adev->common);
4247
4248         list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4249                                  device_node) {
4250                 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4251                 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4252                 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4253                 if (adev->id != PPC440SPE_XOR_ID) {
4254                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4255                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4256                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4257                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4258                         __free_page(ppc440spe_chan->pdest_page);
4259                         __free_page(ppc440spe_chan->qdest_page);
4260                 }
4261                 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4262                                          node) {
4263                         if (ppc440spe_chan ==
4264                             to_ppc440spe_adma_chan(ref->chan)) {
4265                                 list_del(&ref->node);
4266                                 kfree(ref);
4267                         }
4268                 }
4269                 list_del(&chan->device_node);
4270                 kfree(ppc440spe_chan);
4271         }
4272
4273         dma_free_coherent(adev->dev, adev->pool_size,
4274                           adev->dma_desc_pool_virt, adev->dma_desc_pool);
4275         if (adev->id == PPC440SPE_XOR_ID)
4276                 iounmap(adev->xor_reg);
4277         else
4278                 iounmap(adev->dma_reg);
4279         of_address_to_resource(np, 0, &res);
4280         release_mem_region(res.start, resource_size(&res));
4281         kfree(adev);
4282         return 0;
4283 }
4284
4285 /*
4286  * /sys driver interface to enable h/w RAID-6 capabilities
4287  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4288  * directory are "devices", "enable" and "poly".
4289  * "devices" shows available engines.
4290  * "enable" is used to enable RAID-6 capabilities or to check
4291  * whether these has been activated.
4292  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4293  */
4294
4295 static ssize_t devices_show(struct device_driver *dev, char *buf)
4296 {
4297         ssize_t size = 0;
4298         int i;
4299
4300         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4301                 if (ppc440spe_adma_devices[i] == -1)
4302                         continue;
4303                 size += scnprintf(buf + size, PAGE_SIZE - size,
4304                                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4305                                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4306         }
4307         return size;
4308 }
4309 static DRIVER_ATTR_RO(devices);
4310
4311 static ssize_t enable_show(struct device_driver *dev, char *buf)
4312 {
4313         return snprintf(buf, PAGE_SIZE,
4314                         "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4315                         ppc440spe_r6_enabled ? "EN" : "DIS");
4316 }
4317
4318 static ssize_t enable_store(struct device_driver *dev, const char *buf,
4319                             size_t count)
4320 {
4321         unsigned long val;
4322
4323         if (!count || count > 11)
4324                 return -EINVAL;
4325
4326         if (!ppc440spe_r6_tchan)
4327                 return -EFAULT;
4328
4329         /* Write a key */
4330         sscanf(buf, "%lx", &val);
4331         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4332         isync();
4333
4334         /* Verify whether it really works now */
4335         if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4336                 pr_info("PPC440SP(e) RAID-6 has been activated "
4337                         "successfully\n");
4338                 ppc440spe_r6_enabled = 1;
4339         } else {
4340                 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4341                         " Error key ?\n");
4342                 ppc440spe_r6_enabled = 0;
4343         }
4344         return count;
4345 }
4346 static DRIVER_ATTR_RW(enable);
4347
4348 static ssize_t poly_show(struct device_driver *dev, char *buf)
4349 {
4350         ssize_t size = 0;
4351         u32 reg;
4352
4353 #ifdef CONFIG_440SP
4354         /* 440SP has fixed polynomial */
4355         reg = 0x4d;
4356 #else
4357         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4358         reg >>= MQ0_CFBHL_POLY;
4359         reg &= 0xFF;
4360 #endif
4361
4362         size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4363                         "uses 0x1%02x polynomial.\n", reg);
4364         return size;
4365 }
4366
4367 static ssize_t poly_store(struct device_driver *dev, const char *buf,
4368                           size_t count)
4369 {
4370         unsigned long reg, val;
4371
4372 #ifdef CONFIG_440SP
4373         /* 440SP uses default 0x14D polynomial only */
4374         return -EINVAL;
4375 #endif
4376
4377         if (!count || count > 6)
4378                 return -EINVAL;
4379
4380         /* e.g., 0x14D or 0x11D */
4381         sscanf(buf, "%lx", &val);
4382
4383         if (val & ~0x1FF)
4384                 return -EINVAL;
4385
4386         val &= 0xFF;
4387         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4388         reg &= ~(0xFF << MQ0_CFBHL_POLY);
4389         reg |= val << MQ0_CFBHL_POLY;
4390         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4391
4392         return count;
4393 }
4394 static DRIVER_ATTR_RW(poly);
4395
4396 /*
4397  * Common initialisation for RAID engines; allocate memory for
4398  * DMAx FIFOs, perform configuration common for all DMA engines.
4399  * Further DMA engine specific configuration is done at probe time.
4400  */
4401 static int ppc440spe_configure_raid_devices(void)
4402 {
4403         struct device_node *np;
4404         struct resource i2o_res;
4405         struct i2o_regs __iomem *i2o_reg;
4406         dcr_host_t i2o_dcr_host;
4407         unsigned int dcr_base, dcr_len;
4408         int i, ret;
4409
4410         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4411         if (!np) {
4412                 pr_err("%s: can't find I2O device tree node\n",
4413                         __func__);
4414                 return -ENODEV;
4415         }
4416
4417         if (of_address_to_resource(np, 0, &i2o_res)) {
4418                 of_node_put(np);
4419                 return -EINVAL;
4420         }
4421
4422         i2o_reg = of_iomap(np, 0);
4423         if (!i2o_reg) {
4424                 pr_err("%s: failed to map I2O registers\n", __func__);
4425                 of_node_put(np);
4426                 return -EINVAL;
4427         }
4428
4429         /* Get I2O DCRs base */
4430         dcr_base = dcr_resource_start(np, 0);
4431         dcr_len = dcr_resource_len(np, 0);
4432         if (!dcr_base && !dcr_len) {
4433                 pr_err("%pOF: can't get DCR registers base/len!\n", np);
4434                 of_node_put(np);
4435                 iounmap(i2o_reg);
4436                 return -ENODEV;
4437         }
4438
4439         i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4440         if (!DCR_MAP_OK(i2o_dcr_host)) {
4441                 pr_err("%pOF: failed to map DCRs!\n", np);
4442                 of_node_put(np);
4443                 iounmap(i2o_reg);
4444                 return -ENODEV;
4445         }
4446         of_node_put(np);
4447
4448         /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4449          * the base address of FIFO memory space.
4450          * Actually we need twice more physical memory than programmed in the
4451          * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4452          */
4453         ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4454                                          GFP_KERNEL);
4455         if (!ppc440spe_dma_fifo_buf) {
4456                 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4457                 iounmap(i2o_reg);
4458                 dcr_unmap(i2o_dcr_host, dcr_len);
4459                 return -ENOMEM;
4460         }
4461
4462         /*
4463          * Configure h/w
4464          */
4465         /* Reset I2O/DMA */
4466         mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4467         mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4468
4469         /* Setup the base address of mmaped registers */
4470         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4471         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4472                                                 I2O_REG_ENABLE);
4473         dcr_unmap(i2o_dcr_host, dcr_len);
4474
4475         /* Setup FIFO memory space base address */
4476         iowrite32(0, &i2o_reg->ifbah);
4477         iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4478
4479         /* set zero FIFO size for I2O, so the whole
4480          * ppc440spe_dma_fifo_buf is used by DMAs.
4481          * DMAx_FIFOs will be configured while probe.
4482          */
4483         iowrite32(0, &i2o_reg->ifsiz);
4484         iounmap(i2o_reg);
4485
4486         /* To prepare WXOR/RXOR functionality we need access to
4487          * Memory Queue Module DCRs (finally it will be enabled
4488          * via /sys interface of the ppc440spe ADMA driver).
4489          */
4490         np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4491         if (!np) {
4492                 pr_err("%s: can't find MQ device tree node\n",
4493                         __func__);
4494                 ret = -ENODEV;
4495                 goto out_free;
4496         }
4497
4498         /* Get MQ DCRs base */
4499         dcr_base = dcr_resource_start(np, 0);
4500         dcr_len = dcr_resource_len(np, 0);
4501         if (!dcr_base && !dcr_len) {
4502                 pr_err("%pOF: can't get DCR registers base/len!\n", np);
4503                 ret = -ENODEV;
4504                 goto out_mq;
4505         }
4506
4507         ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4508         if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4509                 pr_err("%pOF: failed to map DCRs!\n", np);
4510                 ret = -ENODEV;
4511                 goto out_mq;
4512         }
4513         of_node_put(np);
4514         ppc440spe_mq_dcr_len = dcr_len;
4515
4516         /* Set HB alias */
4517         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4518
4519         /* Set:
4520          * - LL transaction passing limit to 1;
4521          * - Memory controller cycle limit to 1;
4522          * - Galois Polynomial to 0x14d (default)
4523          */
4524         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4525                   (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4526                   (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4527
4528         atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4529         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4530                 ppc440spe_adma_devices[i] = -1;
4531
4532         return 0;
4533
4534 out_mq:
4535         of_node_put(np);
4536 out_free:
4537         kfree(ppc440spe_dma_fifo_buf);
4538         return ret;
4539 }
4540
4541 static const struct of_device_id ppc440spe_adma_of_match[] = {
4542         { .compatible   = "ibm,dma-440spe", },
4543         { .compatible   = "amcc,xor-accelerator", },
4544         {},
4545 };
4546 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4547
4548 static struct platform_driver ppc440spe_adma_driver = {
4549         .probe = ppc440spe_adma_probe,
4550         .remove = ppc440spe_adma_remove,
4551         .driver = {
4552                 .name = "PPC440SP(E)-ADMA",
4553                 .of_match_table = ppc440spe_adma_of_match,
4554         },
4555 };
4556
4557 static __init int ppc440spe_adma_init(void)
4558 {
4559         int ret;
4560
4561         ret = ppc440spe_configure_raid_devices();
4562         if (ret)
4563                 return ret;
4564
4565         ret = platform_driver_register(&ppc440spe_adma_driver);
4566         if (ret) {
4567                 pr_err("%s: failed to register platform driver\n",
4568                         __func__);
4569                 goto out_reg;
4570         }
4571
4572         /* Initialization status */
4573         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4574                                  &driver_attr_devices);
4575         if (ret)
4576                 goto out_dev;
4577
4578         /* RAID-6 h/w enable entry */
4579         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4580                                  &driver_attr_enable);
4581         if (ret)
4582                 goto out_en;
4583
4584         /* GF polynomial to use */
4585         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4586                                  &driver_attr_poly);
4587         if (!ret)
4588                 return ret;
4589
4590         driver_remove_file(&ppc440spe_adma_driver.driver,
4591                            &driver_attr_enable);
4592 out_en:
4593         driver_remove_file(&ppc440spe_adma_driver.driver,
4594                            &driver_attr_devices);
4595 out_dev:
4596         /* User will not be able to enable h/w RAID-6 */
4597         pr_err("%s: failed to create RAID-6 driver interface\n",
4598                 __func__);
4599         platform_driver_unregister(&ppc440spe_adma_driver);
4600 out_reg:
4601         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4602         kfree(ppc440spe_dma_fifo_buf);
4603         return ret;
4604 }
4605
4606 static void __exit ppc440spe_adma_exit(void)
4607 {
4608         driver_remove_file(&ppc440spe_adma_driver.driver,
4609                            &driver_attr_poly);
4610         driver_remove_file(&ppc440spe_adma_driver.driver,
4611                            &driver_attr_enable);
4612         driver_remove_file(&ppc440spe_adma_driver.driver,
4613                            &driver_attr_devices);
4614         platform_driver_unregister(&ppc440spe_adma_driver);
4615         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4616         kfree(ppc440spe_dma_fifo_buf);
4617 }
4618
4619 arch_initcall(ppc440spe_adma_init);
4620 module_exit(ppc440spe_adma_exit);
4621
4622 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4623 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4624 MODULE_LICENSE("GPL");