drivers/dma/ppc4xx/adma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2006-2009 DENX Software Engineering.
   4  *
   5  * Author: Yuri Tikhonov <yur@emcraft.com>
   6  *
   7  * Further porting to arch/powerpc by
   8  *      Anatolij Gustschin <agust@denx.de>
   9  */
  10
  11 /*
  12  * This driver supports the asynchrounous DMA copy and RAID engines available
  13  * on the AMCC PPC440SPe Processors.
  14  * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
  15  * ADMA driver written by D.Williams.
  16  */
  17
  18 #include <linux/init.h>
  19 #include <linux/module.h>
  20 #include <linux/async_tx.h>
  21 #include <linux/delay.h>
  22 #include <linux/dma-mapping.h>
  23 #include <linux/spinlock.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/slab.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/proc_fs.h>
  28 #include <linux/of.h>
  29 #include <linux/of_address.h>
  30 #include <linux/of_irq.h>
  31 #include <linux/of_platform.h>
  32 #include <asm/dcr.h>
  33 #include <asm/dcr-regs.h>
  34 #include "adma.h"
  35 #include "../dmaengine.h"
  36
  37 enum ppc_adma_init_code {
  38         PPC_ADMA_INIT_OK = 0,
  39         PPC_ADMA_INIT_MEMRES,
  40         PPC_ADMA_INIT_MEMREG,
  41         PPC_ADMA_INIT_ALLOC,
  42         PPC_ADMA_INIT_COHERENT,
  43         PPC_ADMA_INIT_CHANNEL,
  44         PPC_ADMA_INIT_IRQ1,
  45         PPC_ADMA_INIT_IRQ2,
  46         PPC_ADMA_INIT_REGISTER
  47 };
  48
  49 static char *ppc_adma_errors[] = {
  50         [PPC_ADMA_INIT_OK] = "ok",
  51         [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource",
  52         [PPC_ADMA_INIT_MEMREG] = "failed to request memory region",
  53         [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev "
  54                                 "structure",
  55         [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for "
  56                                    "hardware descriptors",
  57         [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel",
  58         [PPC_ADMA_INIT_IRQ1] = "failed to request first irq",
  59         [PPC_ADMA_INIT_IRQ2] = "failed to request second irq",
  60         [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device",
  61 };
  62
  63 static enum ppc_adma_init_code
  64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM];
  65
  66 struct ppc_dma_chan_ref {
  67         struct dma_chan *chan;
  68         struct list_head node;
  69 };
  70
  71 /* The list of channels exported by ppc440spe ADMA */
  72 struct list_head
  73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list);
  74
  75 /* This flag is set when want to refetch the xor chain in the interrupt
  76  * handler
  77  */
  78 static u32 do_xor_refetch;
  79
  80 /* Pointer to DMA0, DMA1 CP/CS FIFO */
  81 static void *ppc440spe_dma_fifo_buf;
  82
  83 /* Pointers to last submitted to DMA0, DMA1 CDBs */
  84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3];
  85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3];
  86
  87 /* Pointer to last linked and submitted xor CB */
  88 static struct ppc440spe_adma_desc_slot *xor_last_linked;
  89 static struct ppc440spe_adma_desc_slot *xor_last_submit;
  90
  91 /* This array is used in data-check operations for storing a pattern */
  92 static char ppc440spe_qword[16];
  93
  94 static atomic_t ppc440spe_adma_err_irq_ref;
  95 static dcr_host_t ppc440spe_mq_dcr_host;
  96 static unsigned int ppc440spe_mq_dcr_len;
  97
  98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up
  99  * the block size in transactions, then we do not allow to activate more than
 100  * only one RXOR transactions simultaneously. So use this var to store
 101  * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is
 102  * set) or not (PPC440SPE_RXOR_RUN is clear).
 103  */
 104 static unsigned long ppc440spe_rxor_state;
 105
 106 /* These are used in enable & check routines
 107  */
 108 static u32 ppc440spe_r6_enabled;
 109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan;
 110 static struct completion ppc440spe_r6_test_comp;
 111
 112 static int ppc440spe_adma_dma2rxor_prep_src(
 113                 struct ppc440spe_adma_desc_slot *desc,
 114                 struct ppc440spe_rxor *cursor, int index,
 115                 int src_cnt, u32 addr);
 116 static void ppc440spe_adma_dma2rxor_set_src(
 117                 struct ppc440spe_adma_desc_slot *desc,
 118                 int index, dma_addr_t addr);
 119 static void ppc440spe_adma_dma2rxor_set_mult(
 120                 struct ppc440spe_adma_desc_slot *desc,
 121                 int index, u8 mult);
 122
 123 #ifdef ADMA_LL_DEBUG
 124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; })
 125 #else
 126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; })
 127 #endif
 128
 129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block)
 130 {
 131         struct dma_cdb *cdb;
 132         struct xor_cb *cb;
 133         int i;
 134
 135         switch (chan->device->id) {
 136         case 0:
 137         case 1:
 138                 cdb = block;
 139
 140                 pr_debug("CDB at %p [%d]:\n"
 141                         "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n"
 142                         "\t sg1u 0x%08x sg1l 0x%08x\n"
 143                         "\t sg2u 0x%08x sg2l 0x%08x\n"
 144                         "\t sg3u 0x%08x sg3l 0x%08x\n",
 145                         cdb, chan->device->id,
 146                         cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt),
 147                         le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l),
 148                         le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l),
 149                         le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l)
 150                 );
 151                 break;
 152         case 2:
 153                 cb = block;
 154
 155                 pr_debug("CB at %p [%d]:\n"
 156                         "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n"
 157                         "\t cbtah 0x%08x cbtal 0x%08x\n"
 158                         "\t cblah 0x%08x cblal 0x%08x\n",
 159                         cb, chan->device->id,
 160                         cb->cbc, cb->cbbc, cb->cbs,
 161                         cb->cbtah, cb->cbtal,
 162                         cb->cblah, cb->cblal);
 163                 for (i = 0; i < 16; i++) {
 164                         if (i && !cb->ops[i].h && !cb->ops[i].l)
 165                                 continue;
 166                         pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n",
 167                                 i, cb->ops[i].h, cb->ops[i].l);
 168                 }
 169                 break;
 170         }
 171 }
 172
 173 static void print_cb_list(struct ppc440spe_adma_chan *chan,
 174                           struct ppc440spe_adma_desc_slot *iter)
 175 {
 176         for (; iter; iter = iter->hw_next)
 177                 print_cb(chan, iter->hw_desc);
 178 }
 179
 180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src,
 181                              unsigned int src_cnt)
 182 {
 183         int i;
 184
 185         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 186         for (i = 0; i < src_cnt; i++)
 187                 pr_debug("\t0x%016llx ", src[i]);
 188         pr_debug("dst:\n\t0x%016llx\n", dst);
 189 }
 190
 191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src,
 192                             unsigned int src_cnt)
 193 {
 194         int i;
 195
 196         pr_debug("\n%s(%d):\nsrc: ", __func__, id);
 197         for (i = 0; i < src_cnt; i++)
 198                 pr_debug("\t0x%016llx ", src[i]);
 199         pr_debug("dst: ");
 200         for (i = 0; i < 2; i++)
 201                 pr_debug("\t0x%016llx ", dst[i]);
 202 }
 203
 204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src,
 205                                     unsigned int src_cnt,
 206                                     const unsigned char *scf)
 207 {
 208         int i;
 209
 210         pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id);
 211         if (scf) {
 212                 for (i = 0; i < src_cnt; i++)
 213                         pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]);
 214         } else {
 215                 for (i = 0; i < src_cnt; i++)
 216                         pr_debug("\t0x%016llx(no) ", src[i]);
 217         }
 218
 219         pr_debug("dst: ");
 220         for (i = 0; i < 2; i++)
 221                 pr_debug("\t0x%016llx ", src[src_cnt + i]);
 222 }
 223
 224 /******************************************************************************
 225  * Command (Descriptor) Blocks low-level routines
 226  ******************************************************************************/
 227 /**
 228  * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT
 229  * pseudo operation
 230  */
 231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc,
 232                                           struct ppc440spe_adma_chan *chan)
 233 {
 234         struct xor_cb *p;
 235
 236         switch (chan->device->id) {
 237         case PPC440SPE_XOR_ID:
 238                 p = desc->hw_desc;
 239                 memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 240                 /* NOP with Command Block Complete Enable */
 241                 p->cbc = XOR_CBCR_CBCE_BIT;
 242                 break;
 243         case PPC440SPE_DMA0_ID:
 244         case PPC440SPE_DMA1_ID:
 245                 memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 246                 /* NOP with interrupt */
 247                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 248                 break;
 249         default:
 250                 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id,
 251                                 __func__);
 252                 break;
 253         }
 254 }
 255
 256 /**
 257  * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR
 258  * pseudo operation
 259  */
 260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc)
 261 {
 262         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 263         desc->hw_next = NULL;
 264         desc->src_cnt = 0;
 265         desc->dst_cnt = 1;
 266 }
 267
 268 /**
 269  * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation
 270  */
 271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc,
 272                                          int src_cnt, unsigned long flags)
 273 {
 274         struct xor_cb *hw_desc = desc->hw_desc;
 275
 276         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 277         desc->hw_next = NULL;
 278         desc->src_cnt = src_cnt;
 279         desc->dst_cnt = 1;
 280
 281         hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt;
 282         if (flags & DMA_PREP_INTERRUPT)
 283                 /* Enable interrupt on completion */
 284                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 285 }
 286
 287 /**
 288  * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ
 289  * operation in DMA2 controller
 290  */
 291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc,
 292                 int dst_cnt, int src_cnt, unsigned long flags)
 293 {
 294         struct xor_cb *hw_desc = desc->hw_desc;
 295
 296         memset(desc->hw_desc, 0, sizeof(struct xor_cb));
 297         desc->hw_next = NULL;
 298         desc->src_cnt = src_cnt;
 299         desc->dst_cnt = dst_cnt;
 300         memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags));
 301         desc->descs_per_op = 0;
 302
 303         hw_desc->cbc = XOR_CBCR_TGT_BIT;
 304         if (flags & DMA_PREP_INTERRUPT)
 305                 /* Enable interrupt on completion */
 306                 hw_desc->cbc |= XOR_CBCR_CBCE_BIT;
 307 }
 308
 309 #define DMA_CTRL_FLAGS_LAST     DMA_PREP_FENCE
 310 #define DMA_PREP_ZERO_P         (DMA_CTRL_FLAGS_LAST << 1)
 311 #define DMA_PREP_ZERO_Q         (DMA_PREP_ZERO_P << 1)
 312
 313 /**
 314  * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation
 315  * with DMA0/1
 316  */
 317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc,
 318                                 int dst_cnt, int src_cnt, unsigned long flags,
 319                                 unsigned long op)
 320 {
 321         struct dma_cdb *hw_desc;
 322         struct ppc440spe_adma_desc_slot *iter;
 323         u8 dopc;
 324
 325         /* Common initialization of a PQ descriptors chain */
 326         set_bits(op, &desc->flags);
 327         desc->src_cnt = src_cnt;
 328         desc->dst_cnt = dst_cnt;
 329
 330         /* WXOR MULTICAST if both P and Q are being computed
 331          * MV_SG1_SG2 if Q only
 332          */
 333         dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ?
 334                 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2;
 335
 336         list_for_each_entry(iter, &desc->group_list, chain_node) {
 337                 hw_desc = iter->hw_desc;
 338                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 339
 340                 if (likely(!list_is_last(&iter->chain_node,
 341                                 &desc->group_list))) {
 342                         /* set 'next' pointer */
 343                         iter->hw_next = list_entry(iter->chain_node.next,
 344                                 struct ppc440spe_adma_desc_slot, chain_node);
 345                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 346                 } else {
 347                         /* this is the last descriptor.
 348                          * this slot will be pasted from ADMA level
 349                          * each time it wants to configure parameters
 350                          * of the transaction (src, dst, ...)
 351                          */
 352                         iter->hw_next = NULL;
 353                         if (flags & DMA_PREP_INTERRUPT)
 354                                 set_bit(PPC440SPE_DESC_INT, &iter->flags);
 355                         else
 356                                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
 357                 }
 358         }
 359
 360         /* Set OPS depending on WXOR/RXOR type of operation */
 361         if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) {
 362                 /* This is a WXOR only chain:
 363                  * - first descriptors are for zeroing destinations
 364                  *   if PPC440SPE_ZERO_P/Q set;
 365                  * - descriptors remained are for GF-XOR operations.
 366                  */
 367                 iter = list_first_entry(&desc->group_list,
 368                                         struct ppc440spe_adma_desc_slot,
 369                                         chain_node);
 370
 371                 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) {
 372                         hw_desc = iter->hw_desc;
 373                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 374                         iter = list_first_entry(&iter->chain_node,
 375                                         struct ppc440spe_adma_desc_slot,
 376                                         chain_node);
 377                 }
 378
 379                 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) {
 380                         hw_desc = iter->hw_desc;
 381                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 382                         iter = list_first_entry(&iter->chain_node,
 383                                         struct ppc440spe_adma_desc_slot,
 384                                         chain_node);
 385                 }
 386
 387                 list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 388                         hw_desc = iter->hw_desc;
 389                         hw_desc->opc = dopc;
 390                 }
 391         } else {
 392                 /* This is either RXOR-only or mixed RXOR/WXOR */
 393
 394                 /* The first 1 or 2 slots in chain are always RXOR,
 395                  * if need to calculate P & Q, then there are two
 396                  * RXOR slots; if only P or only Q, then there is one
 397                  */
 398                 iter = list_first_entry(&desc->group_list,
 399                                         struct ppc440spe_adma_desc_slot,
 400                                         chain_node);
 401                 hw_desc = iter->hw_desc;
 402                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 403
 404                 if (desc->dst_cnt == DMA_DEST_MAX_NUM) {
 405                         iter = list_first_entry(&iter->chain_node,
 406                                                 struct ppc440spe_adma_desc_slot,
 407                                                 chain_node);
 408                         hw_desc = iter->hw_desc;
 409                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 410                 }
 411
 412                 /* The remaining descs (if any) are WXORs */
 413                 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) {
 414                         iter = list_first_entry(&iter->chain_node,
 415                                                 struct ppc440spe_adma_desc_slot,
 416                                                 chain_node);
 417                         list_for_each_entry_from(iter, &desc->group_list,
 418                                                 chain_node) {
 419                                 hw_desc = iter->hw_desc;
 420                                 hw_desc->opc = dopc;
 421                         }
 422                 }
 423         }
 424 }
 425
 426 /**
 427  * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor
 428  * for PQ_ZERO_SUM operation
 429  */
 430 static void ppc440spe_desc_init_dma01pqzero_sum(
 431                                 struct ppc440spe_adma_desc_slot *desc,
 432                                 int dst_cnt, int src_cnt)
 433 {
 434         struct dma_cdb *hw_desc;
 435         struct ppc440spe_adma_desc_slot *iter;
 436         int i = 0;
 437         u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST :
 438                                    DMA_CDB_OPC_MV_SG1_SG2;
 439         /*
 440          * Initialize starting from 2nd or 3rd descriptor dependent
 441          * on dst_cnt. First one or two slots are for cloning P
 442          * and/or Q to chan->pdest and/or chan->qdest as we have
 443          * to preserve original P/Q.
 444          */
 445         iter = list_first_entry(&desc->group_list,
 446                                 struct ppc440spe_adma_desc_slot, chain_node);
 447         iter = list_entry(iter->chain_node.next,
 448                           struct ppc440spe_adma_desc_slot, chain_node);
 449
 450         if (dst_cnt > 1) {
 451                 iter = list_entry(iter->chain_node.next,
 452                                   struct ppc440spe_adma_desc_slot, chain_node);
 453         }
 454         /* initialize each source descriptor in chain */
 455         list_for_each_entry_from(iter, &desc->group_list, chain_node) {
 456                 hw_desc = iter->hw_desc;
 457                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
 458                 iter->src_cnt = 0;
 459                 iter->dst_cnt = 0;
 460
 461                 /* This is a ZERO_SUM operation:
 462                  * - <src_cnt> descriptors starting from 2nd or 3rd
 463                  *   descriptor are for GF-XOR operations;
 464                  * - remaining <dst_cnt> descriptors are for checking the result
 465                  */
 466                 if (i++ < src_cnt)
 467                         /* MV_SG1_SG2 if only Q is being verified
 468                          * MULTICAST if both P and Q are being verified
 469                          */
 470                         hw_desc->opc = dopc;
 471                 else
 472                         /* DMA_CDB_OPC_DCHECK128 operation */
 473                         hw_desc->opc = DMA_CDB_OPC_DCHECK128;
 474
 475                 if (likely(!list_is_last(&iter->chain_node,
 476                                          &desc->group_list))) {
 477                         /* set 'next' pointer */
 478                         iter->hw_next = list_entry(iter->chain_node.next,
 479                                                 struct ppc440spe_adma_desc_slot,
 480                                                 chain_node);
 481                 } else {
 482                         /* this is the last descriptor.
 483                          * this slot will be pasted from ADMA level
 484                          * each time it wants to configure parameters
 485                          * of the transaction (src, dst, ...)
 486                          */
 487                         iter->hw_next = NULL;
 488                         /* always enable interrupt generation since we get
 489                          * the status of pqzero from the handler
 490                          */
 491                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
 492                 }
 493         }
 494         desc->src_cnt = src_cnt;
 495         desc->dst_cnt = dst_cnt;
 496 }
 497
 498 /**
 499  * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation
 500  */
 501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc,
 502                                         unsigned long flags)
 503 {
 504         struct dma_cdb *hw_desc = desc->hw_desc;
 505
 506         memset(desc->hw_desc, 0, sizeof(struct dma_cdb));
 507         desc->hw_next = NULL;
 508         desc->src_cnt = 1;
 509         desc->dst_cnt = 1;
 510
 511         if (flags & DMA_PREP_INTERRUPT)
 512                 set_bit(PPC440SPE_DESC_INT, &desc->flags);
 513         else
 514                 clear_bit(PPC440SPE_DESC_INT, &desc->flags);
 515
 516         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
 517 }
 518
 519 /**
 520  * ppc440spe_desc_set_src_addr - set source address into the descriptor
 521  */
 522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc,
 523                                         struct ppc440spe_adma_chan *chan,
 524                                         int src_idx, dma_addr_t addrh,
 525                                         dma_addr_t addrl)
 526 {
 527         struct dma_cdb *dma_hw_desc;
 528         struct xor_cb *xor_hw_desc;
 529         phys_addr_t addr64, tmplow, tmphi;
 530
 531         switch (chan->device->id) {
 532         case PPC440SPE_DMA0_ID:
 533         case PPC440SPE_DMA1_ID:
 534                 if (!addrh) {
 535                         addr64 = addrl;
 536                         tmphi = (addr64 >> 32);
 537                         tmplow = (addr64 & 0xFFFFFFFF);
 538                 } else {
 539                         tmphi = addrh;
 540                         tmplow = addrl;
 541                 }
 542                 dma_hw_desc = desc->hw_desc;
 543                 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow);
 544                 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi);
 545                 break;
 546         case PPC440SPE_XOR_ID:
 547                 xor_hw_desc = desc->hw_desc;
 548                 xor_hw_desc->ops[src_idx].l = addrl;
 549                 xor_hw_desc->ops[src_idx].h |= addrh;
 550                 break;
 551         }
 552 }
 553
 554 /**
 555  * ppc440spe_desc_set_src_mult - set source address mult into the descriptor
 556  */
 557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc,
 558                         struct ppc440spe_adma_chan *chan, u32 mult_index,
 559                         int sg_index, unsigned char mult_value)
 560 {
 561         struct dma_cdb *dma_hw_desc;
 562         struct xor_cb *xor_hw_desc;
 563         u32 *psgu;
 564
 565         switch (chan->device->id) {
 566         case PPC440SPE_DMA0_ID:
 567         case PPC440SPE_DMA1_ID:
 568                 dma_hw_desc = desc->hw_desc;
 569
 570                 switch (sg_index) {
 571                 /* for RXOR operations set multiplier
 572                  * into source cued address
 573                  */
 574                 case DMA_CDB_SG_SRC:
 575                         psgu = &dma_hw_desc->sg1u;
 576                         break;
 577                 /* for WXOR operations set multiplier
 578                  * into destination cued address(es)
 579                  */
 580                 case DMA_CDB_SG_DST1:
 581                         psgu = &dma_hw_desc->sg2u;
 582                         break;
 583                 case DMA_CDB_SG_DST2:
 584                         psgu = &dma_hw_desc->sg3u;
 585                         break;
 586                 default:
 587                         BUG();
 588                 }
 589
 590                 *psgu |= cpu_to_le32(mult_value << mult_index);
 591                 break;
 592         case PPC440SPE_XOR_ID:
 593                 xor_hw_desc = desc->hw_desc;
 594                 break;
 595         default:
 596                 BUG();
 597         }
 598 }
 599
 600 /**
 601  * ppc440spe_desc_set_dest_addr - set destination address into the descriptor
 602  */
 603 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc,
 604                                 struct ppc440spe_adma_chan *chan,
 605                                 dma_addr_t addrh, dma_addr_t addrl,
 606                                 u32 dst_idx)
 607 {
 608         struct dma_cdb *dma_hw_desc;
 609         struct xor_cb *xor_hw_desc;
 610         phys_addr_t addr64, tmphi, tmplow;
 611         u32 *psgu, *psgl;
 612
 613         switch (chan->device->id) {
 614         case PPC440SPE_DMA0_ID:
 615         case PPC440SPE_DMA1_ID:
 616                 if (!addrh) {
 617                         addr64 = addrl;
 618                         tmphi = (addr64 >> 32);
 619                         tmplow = (addr64 & 0xFFFFFFFF);
 620                 } else {
 621                         tmphi = addrh;
 622                         tmplow = addrl;
 623                 }
 624                 dma_hw_desc = desc->hw_desc;
 625
 626                 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u;
 627                 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l;
 628
 629                 *psgl = cpu_to_le32((u32)tmplow);
 630                 *psgu |= cpu_to_le32((u32)tmphi);
 631                 break;
 632         case PPC440SPE_XOR_ID:
 633                 xor_hw_desc = desc->hw_desc;
 634                 xor_hw_desc->cbtal = addrl;
 635                 xor_hw_desc->cbtah |= addrh;
 636                 break;
 637         }
 638 }
 639
 640 /**
 641  * ppc440spe_desc_set_byte_count - set number of data bytes involved
 642  * into the operation
 643  */
 644 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc,
 645                                 struct ppc440spe_adma_chan *chan,
 646                                 u32 byte_count)
 647 {
 648         struct dma_cdb *dma_hw_desc;
 649         struct xor_cb *xor_hw_desc;
 650
 651         switch (chan->device->id) {
 652         case PPC440SPE_DMA0_ID:
 653         case PPC440SPE_DMA1_ID:
 654                 dma_hw_desc = desc->hw_desc;
 655                 dma_hw_desc->cnt = cpu_to_le32(byte_count);
 656                 break;
 657         case PPC440SPE_XOR_ID:
 658                 xor_hw_desc = desc->hw_desc;
 659                 xor_hw_desc->cbbc = byte_count;
 660                 break;
 661         }
 662 }
 663
 664 /**
 665  * ppc440spe_desc_set_rxor_block_size - set RXOR block size
 666  */
 667 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count)
 668 {
 669         /* assume that byte_count is aligned on the 512-boundary;
 670          * thus write it directly to the register (bits 23:31 are
 671          * reserved there).
 672          */
 673         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count);
 674 }
 675
 676 /**
 677  * ppc440spe_desc_set_dcheck - set CHECK pattern
 678  */
 679 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc,
 680                                 struct ppc440spe_adma_chan *chan, u8 *qword)
 681 {
 682         struct dma_cdb *dma_hw_desc;
 683
 684         switch (chan->device->id) {
 685         case PPC440SPE_DMA0_ID:
 686         case PPC440SPE_DMA1_ID:
 687                 dma_hw_desc = desc->hw_desc;
 688                 iowrite32(qword[0], &dma_hw_desc->sg3l);
 689                 iowrite32(qword[4], &dma_hw_desc->sg3u);
 690                 iowrite32(qword[8], &dma_hw_desc->sg2l);
 691                 iowrite32(qword[12], &dma_hw_desc->sg2u);
 692                 break;
 693         default:
 694                 BUG();
 695         }
 696 }
 697
 698 /**
 699  * ppc440spe_xor_set_link - set link address in xor CB
 700  */
 701 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc,
 702                                 struct ppc440spe_adma_desc_slot *next_desc)
 703 {
 704         struct xor_cb *xor_hw_desc = prev_desc->hw_desc;
 705
 706         if (unlikely(!next_desc || !(next_desc->phys))) {
 707                 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n",
 708                         __func__, next_desc,
 709                         next_desc ? next_desc->phys : 0);
 710                 BUG();
 711         }
 712
 713         xor_hw_desc->cbs = 0;
 714         xor_hw_desc->cblal = next_desc->phys;
 715         xor_hw_desc->cblah = 0;
 716         xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT;
 717 }
 718
 719 /**
 720  * ppc440spe_desc_set_link - set the address of descriptor following this
 721  * descriptor in chain
 722  */
 723 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan,
 724                                 struct ppc440spe_adma_desc_slot *prev_desc,
 725                                 struct ppc440spe_adma_desc_slot *next_desc)
 726 {
 727         unsigned long flags;
 728         struct ppc440spe_adma_desc_slot *tail = next_desc;
 729
 730         if (unlikely(!prev_desc || !next_desc ||
 731                 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) {
 732                 /* If previous next is overwritten something is wrong.
 733                  * though we may refetch from append to initiate list
 734                  * processing; in this case - it's ok.
 735                  */
 736                 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; "
 737                         "prev->hw_next=0x%p\n", __func__, prev_desc,
 738                         next_desc, prev_desc ? prev_desc->hw_next : 0);
 739                 BUG();
 740         }
 741
 742         local_irq_save(flags);
 743
 744         /* do s/w chaining both for DMA and XOR descriptors */
 745         prev_desc->hw_next = next_desc;
 746
 747         switch (chan->device->id) {
 748         case PPC440SPE_DMA0_ID:
 749         case PPC440SPE_DMA1_ID:
 750                 break;
 751         case PPC440SPE_XOR_ID:
 752                 /* bind descriptor to the chain */
 753                 while (tail->hw_next)
 754                         tail = tail->hw_next;
 755                 xor_last_linked = tail;
 756
 757                 if (prev_desc == xor_last_submit)
 758                         /* do not link to the last submitted CB */
 759                         break;
 760                 ppc440spe_xor_set_link(prev_desc, next_desc);
 761                 break;
 762         }
 763
 764         local_irq_restore(flags);
 765 }
 766
 767 /**
 768  * ppc440spe_desc_get_link - get the address of the descriptor that
 769  * follows this one
 770  */
 771 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc,
 772                                         struct ppc440spe_adma_chan *chan)
 773 {
 774         if (!desc->hw_next)
 775                 return 0;
 776
 777         return desc->hw_next->phys;
 778 }
 779
 780 /**
 781  * ppc440spe_desc_is_aligned - check alignment
 782  */
 783 static inline int ppc440spe_desc_is_aligned(
 784         struct ppc440spe_adma_desc_slot *desc, int num_slots)
 785 {
 786         return (desc->idx & (num_slots - 1)) ? 0 : 1;
 787 }
 788
 789 /**
 790  * ppc440spe_chan_xor_slot_count - get the number of slots necessary for
 791  * XOR operation
 792  */
 793 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt,
 794                         int *slots_per_op)
 795 {
 796         int slot_cnt;
 797
 798         /* each XOR descriptor provides up to 16 source operands */
 799         slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS;
 800
 801         if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT))
 802                 return slot_cnt;
 803
 804         printk(KERN_ERR "%s: len %d > max %d !!\n",
 805                 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
 806         BUG();
 807         return slot_cnt;
 808 }
 809
 810 /**
 811  * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for
 812  * DMA2 PQ operation
 813  */
 814 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs,
 815                 int src_cnt, size_t len)
 816 {
 817         signed long long order = 0;
 818         int state = 0;
 819         int addr_count = 0;
 820         int i;
 821         for (i = 1; i < src_cnt; i++) {
 822                 dma_addr_t cur_addr = srcs[i];
 823                 dma_addr_t old_addr = srcs[i-1];
 824                 switch (state) {
 825                 case 0:
 826                         if (cur_addr == old_addr + len) {
 827                                 /* direct RXOR */
 828                                 order = 1;
 829                                 state = 1;
 830                                 if (i == src_cnt-1)
 831                                         addr_count++;
 832                         } else if (old_addr == cur_addr + len) {
 833                                 /* reverse RXOR */
 834                                 order = -1;
 835                                 state = 1;
 836                                 if (i == src_cnt-1)
 837                                         addr_count++;
 838                         } else {
 839                                 state = 3;
 840                         }
 841                         break;
 842                 case 1:
 843                         if (i == src_cnt-2 || (order == -1
 844                                 && cur_addr != old_addr - len)) {
 845                                 order = 0;
 846                                 state = 0;
 847                                 addr_count++;
 848                         } else if (cur_addr == old_addr + len*order) {
 849                                 state = 2;
 850                                 if (i == src_cnt-1)
 851                                         addr_count++;
 852                         } else if (cur_addr == old_addr + 2*len) {
 853                                 state = 2;
 854                                 if (i == src_cnt-1)
 855                                         addr_count++;
 856                         } else if (cur_addr == old_addr + 3*len) {
 857                                 state = 2;
 858                                 if (i == src_cnt-1)
 859                                         addr_count++;
 860                         } else {
 861                                 order = 0;
 862                                 state = 0;
 863                                 addr_count++;
 864                         }
 865                         break;
 866                 case 2:
 867                         order = 0;
 868                         state = 0;
 869                         addr_count++;
 870                                 break;
 871                 }
 872                 if (state == 3)
 873                         break;
 874         }
 875         if (src_cnt <= 1 || (state != 1 && state != 2)) {
 876                 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n",
 877                         __func__, src_cnt, state, addr_count, order);
 878                 for (i = 0; i < src_cnt; i++)
 879                         pr_err("\t[%d] 0x%llx \n", i, srcs[i]);
 880                 BUG();
 881         }
 882
 883         return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS;
 884 }
 885
 886
 887 /******************************************************************************
 888  * ADMA channel low-level routines
 889  ******************************************************************************/
 890
 891 static u32
 892 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan);
 893 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan);
 894
 895 /**
 896  * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine
 897  */
 898 static void ppc440spe_adma_device_clear_eot_status(
 899                                         struct ppc440spe_adma_chan *chan)
 900 {
 901         struct dma_regs *dma_reg;
 902         struct xor_regs *xor_reg;
 903         u8 *p = chan->device->dma_desc_pool_virt;
 904         struct dma_cdb *cdb;
 905         u32 rv, i;
 906
 907         switch (chan->device->id) {
 908         case PPC440SPE_DMA0_ID:
 909         case PPC440SPE_DMA1_ID:
 910                 /* read FIFO to ack */
 911                 dma_reg = chan->device->dma_reg;
 912                 while ((rv = ioread32(&dma_reg->csfpl))) {
 913                         i = rv & DMA_CDB_ADDR_MSK;
 914                         cdb = (struct dma_cdb *)&p[i -
 915                             (u32)chan->device->dma_desc_pool];
 916
 917                         /* Clear opcode to ack. This is necessary for
 918                          * ZeroSum operations only
 919                          */
 920                         cdb->opc = 0;
 921
 922                         if (test_bit(PPC440SPE_RXOR_RUN,
 923                             &ppc440spe_rxor_state)) {
 924                                 /* probably this is a completed RXOR op,
 925                                  * get pointer to CDB using the fact that
 926                                  * physical and virtual addresses of CDB
 927                                  * in pools have the same offsets
 928                                  */
 929                                 if (le32_to_cpu(cdb->sg1u) &
 930                                     DMA_CUED_XOR_BASE) {
 931                                         /* this is a RXOR */
 932                                         clear_bit(PPC440SPE_RXOR_RUN,
 933                                                   &ppc440spe_rxor_state);
 934                                 }
 935                         }
 936
 937                         if (rv & DMA_CDB_STATUS_MSK) {
 938                                 /* ZeroSum check failed
 939                                  */
 940                                 struct ppc440spe_adma_desc_slot *iter;
 941                                 dma_addr_t phys = rv & ~DMA_CDB_MSK;
 942
 943                                 /*
 944                                  * Update the status of corresponding
 945                                  * descriptor.
 946                                  */
 947                                 list_for_each_entry(iter, &chan->chain,
 948                                     chain_node) {
 949                                         if (iter->phys == phys)
 950                                                 break;
 951                                 }
 952                                 /*
 953                                  * if cannot find the corresponding
 954                                  * slot it's a bug
 955                                  */
 956                                 BUG_ON(&iter->chain_node == &chan->chain);
 957
 958                                 if (iter->xor_check_result) {
 959                                         if (test_bit(PPC440SPE_DESC_PCHECK,
 960                                                      &iter->flags)) {
 961                                                 *iter->xor_check_result |=
 962                                                         SUM_CHECK_P_RESULT;
 963                                         } else
 964                                         if (test_bit(PPC440SPE_DESC_QCHECK,
 965                                                      &iter->flags)) {
 966                                                 *iter->xor_check_result |=
 967                                                         SUM_CHECK_Q_RESULT;
 968                                         } else
 969                                                 BUG();
 970                                 }
 971                         }
 972                 }
 973
 974                 rv = ioread32(&dma_reg->dsts);
 975                 if (rv) {
 976                         pr_err("DMA%d err status: 0x%x\n",
 977                                chan->device->id, rv);
 978                         /* write back to clear */
 979                         iowrite32(rv, &dma_reg->dsts);
 980                 }
 981                 break;
 982         case PPC440SPE_XOR_ID:
 983                 /* reset status bits to ack */
 984                 xor_reg = chan->device->xor_reg;
 985                 rv = ioread32be(&xor_reg->sr);
 986                 iowrite32be(rv, &xor_reg->sr);
 987
 988                 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) {
 989                         if (rv & XOR_IE_RPTIE_BIT) {
 990                                 /* Read PLB Timeout Error.
 991                                  * Try to resubmit the CB
 992                                  */
 993                                 u32 val = ioread32be(&xor_reg->ccbalr);
 994
 995                                 iowrite32be(val, &xor_reg->cblalr);
 996
 997                                 val = ioread32be(&xor_reg->crsr);
 998                                 iowrite32be(val | XOR_CRSR_XAE_BIT,
 999                                             &xor_reg->crsr);
1000                         } else
1001                                 pr_err("XOR ERR 0x%x status\n", rv);
1002                         break;
1003                 }
1004
1005                 /*  if the XORcore is idle, but there are unprocessed CBs
1006                  * then refetch the s/w chain here
1007                  */
1008                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) &&
1009                     do_xor_refetch)
1010                         ppc440spe_chan_append(chan);
1011                 break;
1012         }
1013 }
1014
1015 /**
1016  * ppc440spe_chan_is_busy - get the channel status
1017  */
1018 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan)
1019 {
1020         struct dma_regs *dma_reg;
1021         struct xor_regs *xor_reg;
1022         int busy = 0;
1023
1024         switch (chan->device->id) {
1025         case PPC440SPE_DMA0_ID:
1026         case PPC440SPE_DMA1_ID:
1027                 dma_reg = chan->device->dma_reg;
1028                 /*  if command FIFO's head and tail pointers are equal and
1029                  * status tail is the same as command, then channel is free
1030                  */
1031                 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) ||
1032                     ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp))
1033                         busy = 1;
1034                 break;
1035         case PPC440SPE_XOR_ID:
1036                 /* use the special status bit for the XORcore
1037                  */
1038                 xor_reg = chan->device->xor_reg;
1039                 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0;
1040                 break;
1041         }
1042
1043         return busy;
1044 }
1045
1046 /**
1047  * ppc440spe_chan_set_first_xor_descriptor -  init XORcore chain
1048  */
1049 static void ppc440spe_chan_set_first_xor_descriptor(
1050                                 struct ppc440spe_adma_chan *chan,
1051                                 struct ppc440spe_adma_desc_slot *next_desc)
1052 {
1053         struct xor_regs *xor_reg = chan->device->xor_reg;
1054
1055         if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)
1056                 printk(KERN_INFO "%s: Warn: XORcore is running "
1057                         "when try to set the first CDB!\n",
1058                         __func__);
1059
1060         xor_last_submit = xor_last_linked = next_desc;
1061
1062         iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr);
1063
1064         iowrite32be(next_desc->phys, &xor_reg->cblalr);
1065         iowrite32be(0, &xor_reg->cblahr);
1066         iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT,
1067                     &xor_reg->cbcr);
1068
1069         chan->hw_chain_inited = 1;
1070 }
1071
1072 /**
1073  * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO.
1074  * called with irqs disabled
1075  */
1076 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan,
1077                 struct ppc440spe_adma_desc_slot *desc)
1078 {
1079         u32 pcdb;
1080         struct dma_regs *dma_reg = chan->device->dma_reg;
1081
1082         pcdb = desc->phys;
1083         if (!test_bit(PPC440SPE_DESC_INT, &desc->flags))
1084                 pcdb |= DMA_CDB_NO_INT;
1085
1086         chan_last_sub[chan->device->id] = desc;
1087
1088         ADMA_LL_DBG(print_cb(chan, desc->hw_desc));
1089
1090         iowrite32(pcdb, &dma_reg->cpfpl);
1091 }
1092
1093 /**
1094  * ppc440spe_chan_append - update the h/w chain in the channel
1095  */
1096 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan)
1097 {
1098         struct xor_regs *xor_reg;
1099         struct ppc440spe_adma_desc_slot *iter;
1100         struct xor_cb *xcb;
1101         u32 cur_desc;
1102         unsigned long flags;
1103
1104         local_irq_save(flags);
1105
1106         switch (chan->device->id) {
1107         case PPC440SPE_DMA0_ID:
1108         case PPC440SPE_DMA1_ID:
1109                 cur_desc = ppc440spe_chan_get_current_descriptor(chan);
1110
1111                 if (likely(cur_desc)) {
1112                         iter = chan_last_sub[chan->device->id];
1113                         BUG_ON(!iter);
1114                 } else {
1115                         /* first peer */
1116                         iter = chan_first_cdb[chan->device->id];
1117                         BUG_ON(!iter);
1118                         ppc440spe_dma_put_desc(chan, iter);
1119                         chan->hw_chain_inited = 1;
1120                 }
1121
1122                 /* is there something new to append */
1123                 if (!iter->hw_next)
1124                         break;
1125
1126                 /* flush descriptors from the s/w queue to fifo */
1127                 list_for_each_entry_continue(iter, &chan->chain, chain_node) {
1128                         ppc440spe_dma_put_desc(chan, iter);
1129                         if (!iter->hw_next)
1130                                 break;
1131                 }
1132                 break;
1133         case PPC440SPE_XOR_ID:
1134                 /* update h/w links and refetch */
1135                 if (!xor_last_submit->hw_next)
1136                         break;
1137
1138                 xor_reg = chan->device->xor_reg;
1139                 /* the last linked CDB has to generate an interrupt
1140                  * that we'd be able to append the next lists to h/w
1141                  * regardless of the XOR engine state at the moment of
1142                  * appending of these next lists
1143                  */
1144                 xcb = xor_last_linked->hw_desc;
1145                 xcb->cbc |= XOR_CBCR_CBCE_BIT;
1146
1147                 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) {
1148                         /* XORcore is idle. Refetch now */
1149                         do_xor_refetch = 0;
1150                         ppc440spe_xor_set_link(xor_last_submit,
1151                                 xor_last_submit->hw_next);
1152
1153                         ADMA_LL_DBG(print_cb_list(chan,
1154                                 xor_last_submit->hw_next));
1155
1156                         xor_last_submit = xor_last_linked;
1157                         iowrite32be(ioread32be(&xor_reg->crsr) |
1158                                     XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT,
1159                                     &xor_reg->crsr);
1160                 } else {
1161                         /* XORcore is running. Refetch later in the handler */
1162                         do_xor_refetch = 1;
1163                 }
1164
1165                 break;
1166         }
1167
1168         local_irq_restore(flags);
1169 }
1170
1171 /**
1172  * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor
1173  */
1174 static u32
1175 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan)
1176 {
1177         struct dma_regs *dma_reg;
1178         struct xor_regs *xor_reg;
1179
1180         if (unlikely(!chan->hw_chain_inited))
1181                 /* h/w descriptor chain is not initialized yet */
1182                 return 0;
1183
1184         switch (chan->device->id) {
1185         case PPC440SPE_DMA0_ID:
1186         case PPC440SPE_DMA1_ID:
1187                 dma_reg = chan->device->dma_reg;
1188                 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK);
1189         case PPC440SPE_XOR_ID:
1190                 xor_reg = chan->device->xor_reg;
1191                 return ioread32be(&xor_reg->ccbalr);
1192         }
1193         return 0;
1194 }
1195
1196 /**
1197  * ppc440spe_chan_run - enable the channel
1198  */
1199 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan)
1200 {
1201         struct xor_regs *xor_reg;
1202
1203         switch (chan->device->id) {
1204         case PPC440SPE_DMA0_ID:
1205         case PPC440SPE_DMA1_ID:
1206                 /* DMAs are always enabled, do nothing */
1207                 break;
1208         case PPC440SPE_XOR_ID:
1209                 /* drain write buffer */
1210                 xor_reg = chan->device->xor_reg;
1211
1212                 /* fetch descriptor pointed to in <link> */
1213                 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT,
1214                             &xor_reg->crsr);
1215                 break;
1216         }
1217 }
1218
1219 /******************************************************************************
1220  * ADMA device level
1221  ******************************************************************************/
1222
1223 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan);
1224 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan);
1225
1226 static dma_cookie_t
1227 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx);
1228
1229 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx,
1230                                     dma_addr_t addr, int index);
1231 static void
1232 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx,
1233                                   dma_addr_t addr, int index);
1234
1235 static void
1236 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx,
1237                            dma_addr_t *paddr, unsigned long flags);
1238 static void
1239 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx,
1240                           dma_addr_t addr, int index);
1241 static void
1242 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx,
1243                                unsigned char mult, int index, int dst_pos);
1244 static void
1245 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx,
1246                                    dma_addr_t paddr, dma_addr_t qaddr);
1247
1248 static struct page *ppc440spe_rxor_srcs[32];
1249
1250 /**
1251  * ppc440spe_can_rxor - check if the operands may be processed with RXOR
1252  */
1253 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len)
1254 {
1255         int i, order = 0, state = 0;
1256         int idx = 0;
1257
1258         if (unlikely(!(src_cnt > 1)))
1259                 return 0;
1260
1261         BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs));
1262
1263         /* Skip holes in the source list before checking */
1264         for (i = 0; i < src_cnt; i++) {
1265                 if (!srcs[i])
1266                         continue;
1267                 ppc440spe_rxor_srcs[idx++] = srcs[i];
1268         }
1269         src_cnt = idx;
1270
1271         for (i = 1; i < src_cnt; i++) {
1272                 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]);
1273                 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]);
1274
1275                 switch (state) {
1276                 case 0:
1277                         if (cur_addr == old_addr + len) {
1278                                 /* direct RXOR */
1279                                 order = 1;
1280                                 state = 1;
1281                         } else if (old_addr == cur_addr + len) {
1282                                 /* reverse RXOR */
1283                                 order = -1;
1284                                 state = 1;
1285                         } else
1286                                 goto out;
1287                         break;
1288                 case 1:
1289                         if ((i == src_cnt - 2) ||
1290                             (order == -1 && cur_addr != old_addr - len)) {
1291                                 order = 0;
1292                                 state = 0;
1293                         } else if ((cur_addr == old_addr + len * order) ||
1294                                    (cur_addr == old_addr + 2 * len) ||
1295                                    (cur_addr == old_addr + 3 * len)) {
1296                                 state = 2;
1297                         } else {
1298                                 order = 0;
1299                                 state = 0;
1300                         }
1301                         break;
1302                 case 2:
1303                         order = 0;
1304                         state = 0;
1305                         break;
1306                 }
1307         }
1308
1309 out:
1310         if (state == 1 || state == 2)
1311                 return 1;
1312
1313         return 0;
1314 }
1315
1316 /**
1317  * ppc440spe_adma_device_estimate - estimate the efficiency of processing
1318  *      the operation given on this channel. It's assumed that 'chan' is
1319  *      capable to process 'cap' type of operation.
1320  * @chan: channel to use
1321  * @cap: type of transaction
1322  * @dst_lst: array of destination pointers
1323  * @dst_cnt: number of destination operands
1324  * @src_lst: array of source pointers
1325  * @src_cnt: number of source operands
1326  * @src_sz: size of each source operand
1327  */
1328 static int ppc440spe_adma_estimate(struct dma_chan *chan,
1329         enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt,
1330         struct page **src_lst, int src_cnt, size_t src_sz)
1331 {
1332         int ef = 1;
1333
1334         if (cap == DMA_PQ || cap == DMA_PQ_VAL) {
1335                 /* If RAID-6 capabilities were not activated don't try
1336                  * to use them
1337                  */
1338                 if (unlikely(!ppc440spe_r6_enabled))
1339                         return -1;
1340         }
1341         /*  In the current implementation of ppc440spe ADMA driver it
1342          * makes sense to pick out only pq case, because it may be
1343          * processed:
1344          * (1) either using Biskup method on DMA2;
1345          * (2) or on DMA0/1.
1346          *  Thus we give a favour to (1) if the sources are suitable;
1347          * else let it be processed on one of the DMA0/1 engines.
1348          *  In the sum_product case where destination is also the
1349          * source process it on DMA0/1 only.
1350          */
1351         if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) {
1352
1353                 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1])
1354                         ef = 0; /* sum_product case, process on DMA0/1 */
1355                 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz))
1356                         ef = 3; /* override (DMA0/1 + idle) */
1357                 else
1358                         ef = 0; /* can't process on DMA2 if !rxor */
1359         }
1360
1361         /* channel idleness increases the priority */
1362         if (likely(ef) &&
1363             !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan)))
1364                 ef++;
1365
1366         return ef;
1367 }
1368
1369 struct dma_chan *
1370 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap,
1371         struct page **dst_lst, int dst_cnt, struct page **src_lst,
1372         int src_cnt, size_t src_sz)
1373 {
1374         struct dma_chan *best_chan = NULL;
1375         struct ppc_dma_chan_ref *ref;
1376         int best_rank = -1;
1377
1378         if (unlikely(!src_sz))
1379                 return NULL;
1380         if (src_sz > PAGE_SIZE) {
1381                 /*
1382                  * should a user of the api ever pass > PAGE_SIZE requests
1383                  * we sort out cases where temporary page-sized buffers
1384                  * are used.
1385                  */
1386                 switch (cap) {
1387                 case DMA_PQ:
1388                         if (src_cnt == 1 && dst_lst[1] == src_lst[0])
1389                                 return NULL;
1390                         if (src_cnt == 2 && dst_lst[1] == src_lst[1])
1391                                 return NULL;
1392                         break;
1393                 case DMA_PQ_VAL:
1394                 case DMA_XOR_VAL:
1395                         return NULL;
1396                 default:
1397                         break;
1398                 }
1399         }
1400
1401         list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) {
1402                 if (dma_has_cap(cap, ref->chan->device->cap_mask)) {
1403                         int rank;
1404
1405                         rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst,
1406                                         dst_cnt, src_lst, src_cnt, src_sz);
1407                         if (rank > best_rank) {
1408                                 best_rank = rank;
1409                                 best_chan = ref->chan;
1410                         }
1411                 }
1412         }
1413
1414         return best_chan;
1415 }
1416 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel);
1417
1418 /**
1419  * ppc440spe_get_group_entry - get group entry with index idx
1420  * @tdesc: is the last allocated slot in the group.
1421  */
1422 static struct ppc440spe_adma_desc_slot *
1423 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx)
1424 {
1425         struct ppc440spe_adma_desc_slot *iter = tdesc->group_head;
1426         int i = 0;
1427
1428         if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) {
1429                 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n",
1430                         __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt);
1431                 BUG();
1432         }
1433
1434         list_for_each_entry(iter, &tdesc->group_list, chain_node) {
1435                 if (i++ == entry_idx)
1436                         break;
1437         }
1438         return iter;
1439 }
1440
1441 /**
1442  * ppc440spe_adma_free_slots - flags descriptor slots for reuse
1443  * @slot: Slot to free
1444  * Caller must hold &ppc440spe_chan->lock while calling this function
1445  */
1446 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot,
1447                                       struct ppc440spe_adma_chan *chan)
1448 {
1449         int stride = slot->slots_per_op;
1450
1451         while (stride--) {
1452                 slot->slots_per_op = 0;
1453                 slot = list_entry(slot->slot_node.next,
1454                                 struct ppc440spe_adma_desc_slot,
1455                                 slot_node);
1456         }
1457 }
1458
1459 /**
1460  * ppc440spe_adma_run_tx_complete_actions - call functions to be called
1461  * upon completion
1462  */
1463 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions(
1464                 struct ppc440spe_adma_desc_slot *desc,
1465                 struct ppc440spe_adma_chan *chan,
1466                 dma_cookie_t cookie)
1467 {
1468         BUG_ON(desc->async_tx.cookie < 0);
1469         if (desc->async_tx.cookie > 0) {
1470                 cookie = desc->async_tx.cookie;
1471                 desc->async_tx.cookie = 0;
1472
1473                 dma_descriptor_unmap(&desc->async_tx);
1474                 /* call the callback (must not sleep or submit new
1475                  * operations to this channel)
1476                  */
1477                 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL);
1478         }
1479
1480         /* run dependent operations */
1481         dma_run_dependencies(&desc->async_tx);
1482
1483         return cookie;
1484 }
1485
1486 /**
1487  * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set)
1488  */
1489 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc,
1490                 struct ppc440spe_adma_chan *chan)
1491 {
1492         /* the client is allowed to attach dependent operations
1493          * until 'ack' is set
1494          */
1495         if (!async_tx_test_ack(&desc->async_tx))
1496                 return 0;
1497
1498         /* leave the last descriptor in the chain
1499          * so we can append to it
1500          */
1501         if (list_is_last(&desc->chain_node, &chan->chain) ||
1502             desc->phys == ppc440spe_chan_get_current_descriptor(chan))
1503                 return 1;
1504
1505         if (chan->device->id != PPC440SPE_XOR_ID) {
1506                 /* our DMA interrupt handler clears opc field of
1507                  * each processed descriptor. For all types of
1508                  * operations except for ZeroSum we do not actually
1509                  * need ack from the interrupt handler. ZeroSum is a
1510                  * special case since the result of this operation
1511                  * is available from the handler only, so if we see
1512                  * such type of descriptor (which is unprocessed yet)
1513                  * then leave it in chain.
1514                  */
1515                 struct dma_cdb *cdb = desc->hw_desc;
1516                 if (cdb->opc == DMA_CDB_OPC_DCHECK128)
1517                         return 1;
1518         }
1519
1520         dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n",
1521                 desc->phys, desc->idx, desc->slots_per_op);
1522
1523         list_del(&desc->chain_node);
1524         ppc440spe_adma_free_slots(desc, chan);
1525         return 0;
1526 }
1527
1528 /**
1529  * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine
1530  *      which runs through the channel CDBs list until reach the descriptor
1531  *      currently processed. When routine determines that all CDBs of group
1532  *      are completed then corresponding callbacks (if any) are called and slots
1533  *      are freed.
1534  */
1535 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1536 {
1537         struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL;
1538         dma_cookie_t cookie = 0;
1539         u32 current_desc = ppc440spe_chan_get_current_descriptor(chan);
1540         int busy = ppc440spe_chan_is_busy(chan);
1541         int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
1542
1543         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n",
1544                 chan->device->id, __func__);
1545
1546         if (!current_desc) {
1547                 /*  There were no transactions yet, so
1548                  * nothing to clean
1549                  */
1550                 return;
1551         }
1552
1553         /* free completed slots from the chain starting with
1554          * the oldest descriptor
1555          */
1556         list_for_each_entry_safe(iter, _iter, &chan->chain,
1557                                         chain_node) {
1558                 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d "
1559                     "busy: %d this_desc: %#llx next_desc: %#x "
1560                     "cur: %#x ack: %d\n",
1561                     iter->async_tx.cookie, iter->idx, busy, iter->phys,
1562                     ppc440spe_desc_get_link(iter, chan), current_desc,
1563                     async_tx_test_ack(&iter->async_tx));
1564                 prefetch(_iter);
1565                 prefetch(&_iter->async_tx);
1566
1567                 /* do not advance past the current descriptor loaded into the
1568                  * hardware channel,subsequent descriptors are either in process
1569                  * or have not been submitted
1570                  */
1571                 if (seen_current)
1572                         break;
1573
1574                 /* stop the search if we reach the current descriptor and the
1575                  * channel is busy, or if it appears that the current descriptor
1576                  * needs to be re-read (i.e. has been appended to)
1577                  */
1578                 if (iter->phys == current_desc) {
1579                         BUG_ON(seen_current++);
1580                         if (busy || ppc440spe_desc_get_link(iter, chan)) {
1581                                 /* not all descriptors of the group have
1582                                  * been completed; exit.
1583                                  */
1584                                 break;
1585                         }
1586                 }
1587
1588                 /* detect the start of a group transaction */
1589                 if (!slot_cnt && !slots_per_op) {
1590                         slot_cnt = iter->slot_cnt;
1591                         slots_per_op = iter->slots_per_op;
1592                         if (slot_cnt <= slots_per_op) {
1593                                 slot_cnt = 0;
1594                                 slots_per_op = 0;
1595                         }
1596                 }
1597
1598                 if (slot_cnt) {
1599                         if (!group_start)
1600                                 group_start = iter;
1601                         slot_cnt -= slots_per_op;
1602                 }
1603
1604                 /* all the members of a group are complete */
1605                 if (slots_per_op != 0 && slot_cnt == 0) {
1606                         struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter;
1607                         int end_of_chain = 0;
1608
1609                         /* clean up the group */
1610                         slot_cnt = group_start->slot_cnt;
1611                         grp_iter = group_start;
1612                         list_for_each_entry_safe_from(grp_iter, _grp_iter,
1613                                 &chan->chain, chain_node) {
1614
1615                                 cookie = ppc440spe_adma_run_tx_complete_actions(
1616                                         grp_iter, chan, cookie);
1617
1618                                 slot_cnt -= slots_per_op;
1619                                 end_of_chain = ppc440spe_adma_clean_slot(
1620                                     grp_iter, chan);
1621                                 if (end_of_chain && slot_cnt) {
1622                                         /* Should wait for ZeroSum completion */
1623                                         if (cookie > 0)
1624                                                 chan->common.completed_cookie = cookie;
1625                                         return;
1626                                 }
1627
1628                                 if (slot_cnt == 0 || end_of_chain)
1629                                         break;
1630                         }
1631
1632                         /* the group should be complete at this point */
1633                         BUG_ON(slot_cnt);
1634
1635                         slots_per_op = 0;
1636                         group_start = NULL;
1637                         if (end_of_chain)
1638                                 break;
1639                         else
1640                                 continue;
1641                 } else if (slots_per_op) /* wait for group completion */
1642                         continue;
1643
1644                 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan,
1645                     cookie);
1646
1647                 if (ppc440spe_adma_clean_slot(iter, chan))
1648                         break;
1649         }
1650
1651         BUG_ON(!seen_current);
1652
1653         if (cookie > 0) {
1654                 chan->common.completed_cookie = cookie;
1655                 pr_debug("\tcompleted cookie %d\n", cookie);
1656         }
1657
1658 }
1659
1660 /**
1661  * ppc440spe_adma_tasklet - clean up watch-dog initiator
1662  */
1663 static void ppc440spe_adma_tasklet(unsigned long data)
1664 {
1665         struct ppc440spe_adma_chan *chan = (struct ppc440spe_adma_chan *) data;
1666
1667         spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING);
1668         __ppc440spe_adma_slot_cleanup(chan);
1669         spin_unlock(&chan->lock);
1670 }
1671
1672 /**
1673  * ppc440spe_adma_slot_cleanup - clean up scheduled initiator
1674  */
1675 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan)
1676 {
1677         spin_lock_bh(&chan->lock);
1678         __ppc440spe_adma_slot_cleanup(chan);
1679         spin_unlock_bh(&chan->lock);
1680 }
1681
1682 /**
1683  * ppc440spe_adma_alloc_slots - allocate free slots (if any)
1684  */
1685 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots(
1686                 struct ppc440spe_adma_chan *chan, int num_slots,
1687                 int slots_per_op)
1688 {
1689         struct ppc440spe_adma_desc_slot *iter = NULL, *_iter;
1690         struct ppc440spe_adma_desc_slot *alloc_start = NULL;
1691         struct list_head chain = LIST_HEAD_INIT(chain);
1692         int slots_found, retry = 0;
1693
1694
1695         BUG_ON(!num_slots || !slots_per_op);
1696         /* start search from the last allocated descrtiptor
1697          * if a contiguous allocation can not be found start searching
1698          * from the beginning of the list
1699          */
1700 retry:
1701         slots_found = 0;
1702         if (retry == 0)
1703                 iter = chan->last_used;
1704         else
1705                 iter = list_entry(&chan->all_slots,
1706                                   struct ppc440spe_adma_desc_slot,
1707                                   slot_node);
1708         list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots,
1709             slot_node) {
1710                 prefetch(_iter);
1711                 prefetch(&_iter->async_tx);
1712                 if (iter->slots_per_op) {
1713                         slots_found = 0;
1714                         continue;
1715                 }
1716
1717                 /* start the allocation if the slot is correctly aligned */
1718                 if (!slots_found++)
1719                         alloc_start = iter;
1720
1721                 if (slots_found == num_slots) {
1722                         struct ppc440spe_adma_desc_slot *alloc_tail = NULL;
1723                         struct ppc440spe_adma_desc_slot *last_used = NULL;
1724
1725                         iter = alloc_start;
1726                         while (num_slots) {
1727                                 int i;
1728                                 /* pre-ack all but the last descriptor */
1729                                 if (num_slots != slots_per_op)
1730                                         async_tx_ack(&iter->async_tx);
1731
1732                                 list_add_tail(&iter->chain_node, &chain);
1733                                 alloc_tail = iter;
1734                                 iter->async_tx.cookie = 0;
1735                                 iter->hw_next = NULL;
1736                                 iter->flags = 0;
1737                                 iter->slot_cnt = num_slots;
1738                                 iter->xor_check_result = NULL;
1739                                 for (i = 0; i < slots_per_op; i++) {
1740                                         iter->slots_per_op = slots_per_op - i;
1741                                         last_used = iter;
1742                                         iter = list_entry(iter->slot_node.next,
1743                                                 struct ppc440spe_adma_desc_slot,
1744                                                 slot_node);
1745                                 }
1746                                 num_slots -= slots_per_op;
1747                         }
1748                         alloc_tail->group_head = alloc_start;
1749                         alloc_tail->async_tx.cookie = -EBUSY;
1750                         list_splice(&chain, &alloc_tail->group_list);
1751                         chan->last_used = last_used;
1752                         return alloc_tail;
1753                 }
1754         }
1755         if (!retry++)
1756                 goto retry;
1757
1758         /* try to free some slots if the allocation fails */
1759         tasklet_schedule(&chan->irq_tasklet);
1760         return NULL;
1761 }
1762
1763 /**
1764  * ppc440spe_adma_alloc_chan_resources -  allocate pools for CDB slots
1765  */
1766 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan)
1767 {
1768         struct ppc440spe_adma_chan *ppc440spe_chan;
1769         struct ppc440spe_adma_desc_slot *slot = NULL;
1770         char *hw_desc;
1771         int i, db_sz;
1772         int init;
1773
1774         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1775         init = ppc440spe_chan->slots_allocated ? 0 : 1;
1776         chan->chan_id = ppc440spe_chan->device->id;
1777
1778         /* Allocate descriptor slots */
1779         i = ppc440spe_chan->slots_allocated;
1780         if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID)
1781                 db_sz = sizeof(struct dma_cdb);
1782         else
1783                 db_sz = sizeof(struct xor_cb);
1784
1785         for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) {
1786                 slot = kzalloc(sizeof(struct ppc440spe_adma_desc_slot),
1787                                GFP_KERNEL);
1788                 if (!slot) {
1789                         printk(KERN_INFO "SPE ADMA Channel only initialized"
1790                                 " %d descriptor slots", i--);
1791                         break;
1792                 }
1793
1794                 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt;
1795                 slot->hw_desc = (void *) &hw_desc[i * db_sz];
1796                 dma_async_tx_descriptor_init(&slot->async_tx, chan);
1797                 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit;
1798                 INIT_LIST_HEAD(&slot->chain_node);
1799                 INIT_LIST_HEAD(&slot->slot_node);
1800                 INIT_LIST_HEAD(&slot->group_list);
1801                 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz;
1802                 slot->idx = i;
1803
1804                 spin_lock_bh(&ppc440spe_chan->lock);
1805                 ppc440spe_chan->slots_allocated++;
1806                 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots);
1807                 spin_unlock_bh(&ppc440spe_chan->lock);
1808         }
1809
1810         if (i && !ppc440spe_chan->last_used) {
1811                 ppc440spe_chan->last_used =
1812                         list_entry(ppc440spe_chan->all_slots.next,
1813                                 struct ppc440spe_adma_desc_slot,
1814                                 slot_node);
1815         }
1816
1817         dev_dbg(ppc440spe_chan->device->common.dev,
1818                 "ppc440spe adma%d: allocated %d descriptor slots\n",
1819                 ppc440spe_chan->device->id, i);
1820
1821         /* initialize the channel and the chain with a null operation */
1822         if (init) {
1823                 switch (ppc440spe_chan->device->id) {
1824                 case PPC440SPE_DMA0_ID:
1825                 case PPC440SPE_DMA1_ID:
1826                         ppc440spe_chan->hw_chain_inited = 0;
1827                         /* Use WXOR for self-testing */
1828                         if (!ppc440spe_r6_tchan)
1829                                 ppc440spe_r6_tchan = ppc440spe_chan;
1830                         break;
1831                 case PPC440SPE_XOR_ID:
1832                         ppc440spe_chan_start_null_xor(ppc440spe_chan);
1833                         break;
1834                 default:
1835                         BUG();
1836                 }
1837                 ppc440spe_chan->needs_unmap = 1;
1838         }
1839
1840         return (i > 0) ? i : -ENOMEM;
1841 }
1842
1843 /**
1844  * ppc440spe_rxor_set_region_data -
1845  */
1846 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc,
1847         u8 xor_arg_no, u32 mask)
1848 {
1849         struct xor_cb *xcb = desc->hw_desc;
1850
1851         xcb->ops[xor_arg_no].h |= mask;
1852 }
1853
1854 /**
1855  * ppc440spe_rxor_set_src -
1856  */
1857 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc,
1858         u8 xor_arg_no, dma_addr_t addr)
1859 {
1860         struct xor_cb *xcb = desc->hw_desc;
1861
1862         xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE;
1863         xcb->ops[xor_arg_no].l = addr;
1864 }
1865
1866 /**
1867  * ppc440spe_rxor_set_mult -
1868  */
1869 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc,
1870         u8 xor_arg_no, u8 idx, u8 mult)
1871 {
1872         struct xor_cb *xcb = desc->hw_desc;
1873
1874         xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8);
1875 }
1876
1877 /**
1878  * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold
1879  *      has been achieved
1880  */
1881 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan)
1882 {
1883         dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n",
1884                 chan->device->id, chan->pending);
1885
1886         if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) {
1887                 chan->pending = 0;
1888                 ppc440spe_chan_append(chan);
1889         }
1890 }
1891
1892 /**
1893  * ppc440spe_adma_tx_submit - submit new descriptor group to the channel
1894  *      (it's not necessary that descriptors will be submitted to the h/w
1895  *      chains too right now)
1896  */
1897 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx)
1898 {
1899         struct ppc440spe_adma_desc_slot *sw_desc;
1900         struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan);
1901         struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail;
1902         int slot_cnt;
1903         int slots_per_op;
1904         dma_cookie_t cookie;
1905
1906         sw_desc = tx_to_ppc440spe_adma_slot(tx);
1907
1908         group_start = sw_desc->group_head;
1909         slot_cnt = group_start->slot_cnt;
1910         slots_per_op = group_start->slots_per_op;
1911
1912         spin_lock_bh(&chan->lock);
1913         cookie = dma_cookie_assign(tx);
1914
1915         if (unlikely(list_empty(&chan->chain))) {
1916                 /* first peer */
1917                 list_splice_init(&sw_desc->group_list, &chan->chain);
1918                 chan_first_cdb[chan->device->id] = group_start;
1919         } else {
1920                 /* isn't first peer, bind CDBs to chain */
1921                 old_chain_tail = list_entry(chan->chain.prev,
1922                                         struct ppc440spe_adma_desc_slot,
1923                                         chain_node);
1924                 list_splice_init(&sw_desc->group_list,
1925                     &old_chain_tail->chain_node);
1926                 /* fix up the hardware chain */
1927                 ppc440spe_desc_set_link(chan, old_chain_tail, group_start);
1928         }
1929
1930         /* increment the pending count by the number of operations */
1931         chan->pending += slot_cnt / slots_per_op;
1932         ppc440spe_adma_check_threshold(chan);
1933         spin_unlock_bh(&chan->lock);
1934
1935         dev_dbg(chan->device->common.dev,
1936                 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n",
1937                 chan->device->id, __func__,
1938                 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc);
1939
1940         return cookie;
1941 }
1942
1943 /**
1944  * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation
1945  */
1946 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt(
1947                 struct dma_chan *chan, unsigned long flags)
1948 {
1949         struct ppc440spe_adma_chan *ppc440spe_chan;
1950         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1951         int slot_cnt, slots_per_op;
1952
1953         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1954
1955         dev_dbg(ppc440spe_chan->device->common.dev,
1956                 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id,
1957                 __func__);
1958
1959         spin_lock_bh(&ppc440spe_chan->lock);
1960         slot_cnt = slots_per_op = 1;
1961         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
1962                         slots_per_op);
1963         if (sw_desc) {
1964                 group_start = sw_desc->group_head;
1965                 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan);
1966                 group_start->unmap_len = 0;
1967                 sw_desc->async_tx.flags = flags;
1968         }
1969         spin_unlock_bh(&ppc440spe_chan->lock);
1970
1971         return sw_desc ? &sw_desc->async_tx : NULL;
1972 }
1973
1974 /**
1975  * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation
1976  */
1977 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy(
1978                 struct dma_chan *chan, dma_addr_t dma_dest,
1979                 dma_addr_t dma_src, size_t len, unsigned long flags)
1980 {
1981         struct ppc440spe_adma_chan *ppc440spe_chan;
1982         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
1983         int slot_cnt, slots_per_op;
1984
1985         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
1986
1987         if (unlikely(!len))
1988                 return NULL;
1989
1990         BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT);
1991
1992         spin_lock_bh(&ppc440spe_chan->lock);
1993
1994         dev_dbg(ppc440spe_chan->device->common.dev,
1995                 "ppc440spe adma%d: %s len: %u int_en %d\n",
1996                 ppc440spe_chan->device->id, __func__, len,
1997                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
1998         slot_cnt = slots_per_op = 1;
1999         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2000                 slots_per_op);
2001         if (sw_desc) {
2002                 group_start = sw_desc->group_head;
2003                 ppc440spe_desc_init_memcpy(group_start, flags);
2004                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2005                 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0);
2006                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2007                 sw_desc->unmap_len = len;
2008                 sw_desc->async_tx.flags = flags;
2009         }
2010         spin_unlock_bh(&ppc440spe_chan->lock);
2011
2012         return sw_desc ? &sw_desc->async_tx : NULL;
2013 }
2014
2015 /**
2016  * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation
2017  */
2018 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor(
2019                 struct dma_chan *chan, dma_addr_t dma_dest,
2020                 dma_addr_t *dma_src, u32 src_cnt, size_t len,
2021                 unsigned long flags)
2022 {
2023         struct ppc440spe_adma_chan *ppc440spe_chan;
2024         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
2025         int slot_cnt, slots_per_op;
2026
2027         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2028
2029         ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id,
2030                                      dma_dest, dma_src, src_cnt));
2031         if (unlikely(!len))
2032                 return NULL;
2033         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2034
2035         dev_dbg(ppc440spe_chan->device->common.dev,
2036                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2037                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2038                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2039
2040         spin_lock_bh(&ppc440spe_chan->lock);
2041         slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op);
2042         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2043                         slots_per_op);
2044         if (sw_desc) {
2045                 group_start = sw_desc->group_head;
2046                 ppc440spe_desc_init_xor(group_start, src_cnt, flags);
2047                 ppc440spe_adma_set_dest(group_start, dma_dest, 0);
2048                 while (src_cnt--)
2049                         ppc440spe_adma_memcpy_xor_set_src(group_start,
2050                                 dma_src[src_cnt], src_cnt);
2051                 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len);
2052                 sw_desc->unmap_len = len;
2053                 sw_desc->async_tx.flags = flags;
2054         }
2055         spin_unlock_bh(&ppc440spe_chan->lock);
2056
2057         return sw_desc ? &sw_desc->async_tx : NULL;
2058 }
2059
2060 static inline void
2061 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc,
2062                                 int src_cnt);
2063 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor);
2064
2065 /**
2066  * ppc440spe_adma_init_dma2rxor_slot -
2067  */
2068 static void ppc440spe_adma_init_dma2rxor_slot(
2069                 struct ppc440spe_adma_desc_slot *desc,
2070                 dma_addr_t *src, int src_cnt)
2071 {
2072         int i;
2073
2074         /* initialize CDB */
2075         for (i = 0; i < src_cnt; i++) {
2076                 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i,
2077                                                  desc->src_cnt, (u32)src[i]);
2078         }
2079 }
2080
2081 /**
2082  * ppc440spe_dma01_prep_mult -
2083  * for Q operation where destination is also the source
2084  */
2085 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult(
2086                 struct ppc440spe_adma_chan *ppc440spe_chan,
2087                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2088                 const unsigned char *scf, size_t len, unsigned long flags)
2089 {
2090         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2091         unsigned long op = 0;
2092         int slot_cnt;
2093
2094         set_bit(PPC440SPE_DESC_WXOR, &op);
2095         slot_cnt = 2;
2096
2097         spin_lock_bh(&ppc440spe_chan->lock);
2098
2099         /* use WXOR, each descriptor occupies one slot */
2100         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2101         if (sw_desc) {
2102                 struct ppc440spe_adma_chan *chan;
2103                 struct ppc440spe_adma_desc_slot *iter;
2104                 struct dma_cdb *hw_desc;
2105
2106                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2107                 set_bits(op, &sw_desc->flags);
2108                 sw_desc->src_cnt = src_cnt;
2109                 sw_desc->dst_cnt = dst_cnt;
2110                 /* First descriptor, zero data in the destination and copy it
2111                  * to q page using MULTICAST transfer.
2112                  */
2113                 iter = list_first_entry(&sw_desc->group_list,
2114                                         struct ppc440spe_adma_desc_slot,
2115                                         chain_node);
2116                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2117                 /* set 'next' pointer */
2118                 iter->hw_next = list_entry(iter->chain_node.next,
2119                                            struct ppc440spe_adma_desc_slot,
2120                                            chain_node);
2121                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2122                 hw_desc = iter->hw_desc;
2123                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2124
2125                 ppc440spe_desc_set_dest_addr(iter, chan,
2126                                              DMA_CUED_XOR_BASE, dst[0], 0);
2127                 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1);
2128                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2129                                             src[0]);
2130                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2131                 iter->unmap_len = len;
2132
2133                 /*
2134                  * Second descriptor, multiply data from the q page
2135                  * and store the result in real destination.
2136                  */
2137                 iter = list_first_entry(&iter->chain_node,
2138                                         struct ppc440spe_adma_desc_slot,
2139                                         chain_node);
2140                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2141                 iter->hw_next = NULL;
2142                 if (flags & DMA_PREP_INTERRUPT)
2143                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2144                 else
2145                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2146
2147                 hw_desc = iter->hw_desc;
2148                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2149                 ppc440spe_desc_set_src_addr(iter, chan, 0,
2150                                             DMA_CUED_XOR_HB, dst[1]);
2151                 ppc440spe_desc_set_dest_addr(iter, chan,
2152                                              DMA_CUED_XOR_BASE, dst[0], 0);
2153
2154                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2155                                             DMA_CDB_SG_DST1, scf[0]);
2156                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2157                 iter->unmap_len = len;
2158                 sw_desc->async_tx.flags = flags;
2159         }
2160
2161         spin_unlock_bh(&ppc440spe_chan->lock);
2162
2163         return sw_desc;
2164 }
2165
2166 /**
2167  * ppc440spe_dma01_prep_sum_product -
2168  * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also
2169  * the source.
2170  */
2171 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product(
2172                 struct ppc440spe_adma_chan *ppc440spe_chan,
2173                 dma_addr_t *dst, dma_addr_t *src, int src_cnt,
2174                 const unsigned char *scf, size_t len, unsigned long flags)
2175 {
2176         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2177         unsigned long op = 0;
2178         int slot_cnt;
2179
2180         set_bit(PPC440SPE_DESC_WXOR, &op);
2181         slot_cnt = 3;
2182
2183         spin_lock_bh(&ppc440spe_chan->lock);
2184
2185         /* WXOR, each descriptor occupies one slot */
2186         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2187         if (sw_desc) {
2188                 struct ppc440spe_adma_chan *chan;
2189                 struct ppc440spe_adma_desc_slot *iter;
2190                 struct dma_cdb *hw_desc;
2191
2192                 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2193                 set_bits(op, &sw_desc->flags);
2194                 sw_desc->src_cnt = src_cnt;
2195                 sw_desc->dst_cnt = 1;
2196                 /* 1st descriptor, src[1] data to q page and zero destination */
2197                 iter = list_first_entry(&sw_desc->group_list,
2198                                         struct ppc440spe_adma_desc_slot,
2199                                         chain_node);
2200                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2201                 iter->hw_next = list_entry(iter->chain_node.next,
2202                                            struct ppc440spe_adma_desc_slot,
2203                                            chain_node);
2204                 clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2205                 hw_desc = iter->hw_desc;
2206                 hw_desc->opc = DMA_CDB_OPC_MULTICAST;
2207
2208                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2209                                              *dst, 0);
2210                 ppc440spe_desc_set_dest_addr(iter, chan, 0,
2211                                              ppc440spe_chan->qdest, 1);
2212                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2213                                             src[1]);
2214                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2215                 iter->unmap_len = len;
2216
2217                 /* 2nd descriptor, multiply src[1] data and store the
2218                  * result in destination */
2219                 iter = list_first_entry(&iter->chain_node,
2220                                         struct ppc440spe_adma_desc_slot,
2221                                         chain_node);
2222                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2223                 /* set 'next' pointer */
2224                 iter->hw_next = list_entry(iter->chain_node.next,
2225                                            struct ppc440spe_adma_desc_slot,
2226                                            chain_node);
2227                 if (flags & DMA_PREP_INTERRUPT)
2228                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2229                 else
2230                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2231
2232                 hw_desc = iter->hw_desc;
2233                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2234                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2235                                             ppc440spe_chan->qdest);
2236                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2237                                              *dst, 0);
2238                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2239                                             DMA_CDB_SG_DST1, scf[1]);
2240                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2241                 iter->unmap_len = len;
2242
2243                 /*
2244                  * 3rd descriptor, multiply src[0] data and xor it
2245                  * with destination
2246                  */
2247                 iter = list_first_entry(&iter->chain_node,
2248                                         struct ppc440spe_adma_desc_slot,
2249                                         chain_node);
2250                 memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2251                 iter->hw_next = NULL;
2252                 if (flags & DMA_PREP_INTERRUPT)
2253                         set_bit(PPC440SPE_DESC_INT, &iter->flags);
2254                 else
2255                         clear_bit(PPC440SPE_DESC_INT, &iter->flags);
2256
2257                 hw_desc = iter->hw_desc;
2258                 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2259                 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB,
2260                                             src[0]);
2261                 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE,
2262                                              *dst, 0);
2263                 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2264                                             DMA_CDB_SG_DST1, scf[0]);
2265                 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len);
2266                 iter->unmap_len = len;
2267                 sw_desc->async_tx.flags = flags;
2268         }
2269
2270         spin_unlock_bh(&ppc440spe_chan->lock);
2271
2272         return sw_desc;
2273 }
2274
2275 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq(
2276                 struct ppc440spe_adma_chan *ppc440spe_chan,
2277                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2278                 const unsigned char *scf, size_t len, unsigned long flags)
2279 {
2280         int slot_cnt;
2281         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2282         unsigned long op = 0;
2283         unsigned char mult = 1;
2284
2285         pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2286                  __func__, dst_cnt, src_cnt, len);
2287         /*  select operations WXOR/RXOR depending on the
2288          * source addresses of operators and the number
2289          * of destinations (RXOR support only Q-parity calculations)
2290          */
2291         set_bit(PPC440SPE_DESC_WXOR, &op);
2292         if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) {
2293                 /* no active RXOR;
2294                  * do RXOR if:
2295                  * - there are more than 1 source,
2296                  * - len is aligned on 512-byte boundary,
2297                  * - source addresses fit to one of 4 possible regions.
2298                  */
2299                 if (src_cnt > 1 &&
2300                     !(len & MQ0_CF2H_RXOR_BS_MASK) &&
2301                     (src[0] + len) == src[1]) {
2302                         /* may do RXOR R1 R2 */
2303                         set_bit(PPC440SPE_DESC_RXOR, &op);
2304                         if (src_cnt != 2) {
2305                                 /* may try to enhance region of RXOR */
2306                                 if ((src[1] + len) == src[2]) {
2307                                         /* do RXOR R1 R2 R3 */
2308                                         set_bit(PPC440SPE_DESC_RXOR123,
2309                                                 &op);
2310                                 } else if ((src[1] + len * 2) == src[2]) {
2311                                         /* do RXOR R1 R2 R4 */
2312                                         set_bit(PPC440SPE_DESC_RXOR124, &op);
2313                                 } else if ((src[1] + len * 3) == src[2]) {
2314                                         /* do RXOR R1 R2 R5 */
2315                                         set_bit(PPC440SPE_DESC_RXOR125,
2316                                                 &op);
2317                                 } else {
2318                                         /* do RXOR R1 R2 */
2319                                         set_bit(PPC440SPE_DESC_RXOR12,
2320                                                 &op);
2321                                 }
2322                         } else {
2323                                 /* do RXOR R1 R2 */
2324                                 set_bit(PPC440SPE_DESC_RXOR12, &op);
2325                         }
2326                 }
2327
2328                 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2329                         /* can not do this operation with RXOR */
2330                         clear_bit(PPC440SPE_RXOR_RUN,
2331                                 &ppc440spe_rxor_state);
2332                 } else {
2333                         /* can do; set block size right now */
2334                         ppc440spe_desc_set_rxor_block_size(len);
2335                 }
2336         }
2337
2338         /* Number of necessary slots depends on operation type selected */
2339         if (!test_bit(PPC440SPE_DESC_RXOR, &op)) {
2340                 /*  This is a WXOR only chain. Need descriptors for each
2341                  * source to GF-XOR them with WXOR, and need descriptors
2342                  * for each destination to zero them with WXOR
2343                  */
2344                 slot_cnt = src_cnt;
2345
2346                 if (flags & DMA_PREP_ZERO_P) {
2347                         slot_cnt++;
2348                         set_bit(PPC440SPE_ZERO_P, &op);
2349                 }
2350                 if (flags & DMA_PREP_ZERO_Q) {
2351                         slot_cnt++;
2352                         set_bit(PPC440SPE_ZERO_Q, &op);
2353                 }
2354         } else {
2355                 /*  Need 1/2 descriptor for RXOR operation, and
2356                  * need (src_cnt - (2 or 3)) for WXOR of sources
2357                  * remained (if any)
2358                  */
2359                 slot_cnt = dst_cnt;
2360
2361                 if (flags & DMA_PREP_ZERO_P)
2362                         set_bit(PPC440SPE_ZERO_P, &op);
2363                 if (flags & DMA_PREP_ZERO_Q)
2364                         set_bit(PPC440SPE_ZERO_Q, &op);
2365
2366                 if (test_bit(PPC440SPE_DESC_RXOR12, &op))
2367                         slot_cnt += src_cnt - 2;
2368                 else
2369                         slot_cnt += src_cnt - 3;
2370
2371                 /*  Thus we have either RXOR only chain or
2372                  * mixed RXOR/WXOR
2373                  */
2374                 if (slot_cnt == dst_cnt)
2375                         /* RXOR only chain */
2376                         clear_bit(PPC440SPE_DESC_WXOR, &op);
2377         }
2378
2379         spin_lock_bh(&ppc440spe_chan->lock);
2380         /* for both RXOR/WXOR each descriptor occupies one slot */
2381         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2382         if (sw_desc) {
2383                 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt,
2384                                 flags, op);
2385
2386                 /* setup dst/src/mult */
2387                 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n",
2388                          __func__, dst[0], dst[1]);
2389                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2390                 while (src_cnt--) {
2391                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2392                                                   src_cnt);
2393
2394                         /* NOTE: "Multi = 0 is equivalent to = 1" as it
2395                          * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf
2396                          * doesn't work for RXOR with DMA0/1! Instead, multi=0
2397                          * leads to zeroing source data after RXOR.
2398                          * So, for P case set-up mult=1 explicitly.
2399                          */
2400                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2401                                 mult = scf[src_cnt];
2402                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2403                                 mult, src_cnt,  dst_cnt - 1);
2404                 }
2405
2406                 /* Setup byte count foreach slot just allocated */
2407                 sw_desc->async_tx.flags = flags;
2408                 list_for_each_entry(iter, &sw_desc->group_list,
2409                                 chain_node) {
2410                         ppc440spe_desc_set_byte_count(iter,
2411                                 ppc440spe_chan, len);
2412                         iter->unmap_len = len;
2413                 }
2414         }
2415         spin_unlock_bh(&ppc440spe_chan->lock);
2416
2417         return sw_desc;
2418 }
2419
2420 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq(
2421                 struct ppc440spe_adma_chan *ppc440spe_chan,
2422                 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt,
2423                 const unsigned char *scf, size_t len, unsigned long flags)
2424 {
2425         int slot_cnt, descs_per_op;
2426         struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter;
2427         unsigned long op = 0;
2428         unsigned char mult = 1;
2429
2430         BUG_ON(!dst_cnt);
2431         /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n",
2432                  __func__, dst_cnt, src_cnt, len);*/
2433
2434         spin_lock_bh(&ppc440spe_chan->lock);
2435         descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len);
2436         if (descs_per_op < 0) {
2437                 spin_unlock_bh(&ppc440spe_chan->lock);
2438                 return NULL;
2439         }
2440
2441         /* depending on number of sources we have 1 or 2 RXOR chains */
2442         slot_cnt = descs_per_op * dst_cnt;
2443
2444         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1);
2445         if (sw_desc) {
2446                 op = slot_cnt;
2447                 sw_desc->async_tx.flags = flags;
2448                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2449                         ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt,
2450                                 --op ? 0 : flags);
2451                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2452                                 len);
2453                         iter->unmap_len = len;
2454
2455                         ppc440spe_init_rxor_cursor(&(iter->rxor_cursor));
2456                         iter->rxor_cursor.len = len;
2457                         iter->descs_per_op = descs_per_op;
2458                 }
2459                 op = 0;
2460                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2461                         op++;
2462                         if (op % descs_per_op == 0)
2463                                 ppc440spe_adma_init_dma2rxor_slot(iter, src,
2464                                                                   src_cnt);
2465                         if (likely(!list_is_last(&iter->chain_node,
2466                                                  &sw_desc->group_list))) {
2467                                 /* set 'next' pointer */
2468                                 iter->hw_next =
2469                                         list_entry(iter->chain_node.next,
2470                                                 struct ppc440spe_adma_desc_slot,
2471                                                 chain_node);
2472                                 ppc440spe_xor_set_link(iter, iter->hw_next);
2473                         } else {
2474                                 /* this is the last descriptor. */
2475                                 iter->hw_next = NULL;
2476                         }
2477                 }
2478
2479                 /* fixup head descriptor */
2480                 sw_desc->dst_cnt = dst_cnt;
2481                 if (flags & DMA_PREP_ZERO_P)
2482                         set_bit(PPC440SPE_ZERO_P, &sw_desc->flags);
2483                 if (flags & DMA_PREP_ZERO_Q)
2484                         set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags);
2485
2486                 /* setup dst/src/mult */
2487                 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags);
2488
2489                 while (src_cnt--) {
2490                         /* handle descriptors (if dst_cnt == 2) inside
2491                          * the ppc440spe_adma_pq_set_srcxxx() functions
2492                          */
2493                         ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt],
2494                                                   src_cnt);
2495                         if (!(flags & DMA_PREP_PQ_DISABLE_Q))
2496                                 mult = scf[src_cnt];
2497                         ppc440spe_adma_pq_set_src_mult(sw_desc,
2498                                         mult, src_cnt, dst_cnt - 1);
2499                 }
2500         }
2501         spin_unlock_bh(&ppc440spe_chan->lock);
2502         ppc440spe_desc_set_rxor_block_size(len);
2503         return sw_desc;
2504 }
2505
2506 /**
2507  * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation
2508  */
2509 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq(
2510                 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
2511                 unsigned int src_cnt, const unsigned char *scf,
2512                 size_t len, unsigned long flags)
2513 {
2514         struct ppc440spe_adma_chan *ppc440spe_chan;
2515         struct ppc440spe_adma_desc_slot *sw_desc = NULL;
2516         int dst_cnt = 0;
2517
2518         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2519
2520         ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id,
2521                                     dst, src, src_cnt));
2522         BUG_ON(!len);
2523         BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT);
2524         BUG_ON(!src_cnt);
2525
2526         if (src_cnt == 1 && dst[1] == src[0]) {
2527                 dma_addr_t dest[2];
2528
2529                 /* dst[1] is real destination (Q) */
2530                 dest[0] = dst[1];
2531                 /* this is the page to multicast source data to */
2532                 dest[1] = ppc440spe_chan->qdest;
2533                 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan,
2534                                 dest, 2, src, src_cnt, scf, len, flags);
2535                 return sw_desc ? &sw_desc->async_tx : NULL;
2536         }
2537
2538         if (src_cnt == 2 && dst[1] == src[1]) {
2539                 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan,
2540                                         &dst[1], src, 2, scf, len, flags);
2541                 return sw_desc ? &sw_desc->async_tx : NULL;
2542         }
2543
2544         if (!(flags & DMA_PREP_PQ_DISABLE_P)) {
2545                 BUG_ON(!dst[0]);
2546                 dst_cnt++;
2547                 flags |= DMA_PREP_ZERO_P;
2548         }
2549
2550         if (!(flags & DMA_PREP_PQ_DISABLE_Q)) {
2551                 BUG_ON(!dst[1]);
2552                 dst_cnt++;
2553                 flags |= DMA_PREP_ZERO_Q;
2554         }
2555
2556         BUG_ON(!dst_cnt);
2557
2558         dev_dbg(ppc440spe_chan->device->common.dev,
2559                 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n",
2560                 ppc440spe_chan->device->id, __func__, src_cnt, len,
2561                 flags & DMA_PREP_INTERRUPT ? 1 : 0);
2562
2563         switch (ppc440spe_chan->device->id) {
2564         case PPC440SPE_DMA0_ID:
2565         case PPC440SPE_DMA1_ID:
2566                 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan,
2567                                 dst, dst_cnt, src, src_cnt, scf,
2568                                 len, flags);
2569                 break;
2570
2571         case PPC440SPE_XOR_ID:
2572                 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan,
2573                                 dst, dst_cnt, src, src_cnt, scf,
2574                                 len, flags);
2575                 break;
2576         }
2577
2578         return sw_desc ? &sw_desc->async_tx : NULL;
2579 }
2580
2581 /**
2582  * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for
2583  * a PQ_ZERO_SUM operation
2584  */
2585 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum(
2586                 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
2587                 unsigned int src_cnt, const unsigned char *scf, size_t len,
2588                 enum sum_check_flags *pqres, unsigned long flags)
2589 {
2590         struct ppc440spe_adma_chan *ppc440spe_chan;
2591         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
2592         dma_addr_t pdest, qdest;
2593         int slot_cnt, slots_per_op, idst, dst_cnt;
2594
2595         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
2596
2597         if (flags & DMA_PREP_PQ_DISABLE_P)
2598                 pdest = 0;
2599         else
2600                 pdest = pq[0];
2601
2602         if (flags & DMA_PREP_PQ_DISABLE_Q)
2603                 qdest = 0;
2604         else
2605                 qdest = pq[1];
2606
2607         ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id,
2608                                             src, src_cnt, scf));
2609
2610         /* Always use WXOR for P/Q calculations (two destinations).
2611          * Need 1 or 2 extra slots to verify results are zero.
2612          */
2613         idst = dst_cnt = (pdest && qdest) ? 2 : 1;
2614
2615         /* One additional slot per destination to clone P/Q
2616          * before calculation (we have to preserve destinations).
2617          */
2618         slot_cnt = src_cnt + dst_cnt * 2;
2619         slots_per_op = 1;
2620
2621         spin_lock_bh(&ppc440spe_chan->lock);
2622         sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt,
2623                                              slots_per_op);
2624         if (sw_desc) {
2625                 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt);
2626
2627                 /* Setup byte count for each slot just allocated */
2628                 sw_desc->async_tx.flags = flags;
2629                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
2630                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2631                                                       len);
2632                         iter->unmap_len = len;
2633                 }
2634
2635                 if (pdest) {
2636                         struct dma_cdb *hw_desc;
2637                         struct ppc440spe_adma_chan *chan;
2638
2639                         iter = sw_desc->group_head;
2640                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2641                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2642                         iter->hw_next = list_entry(iter->chain_node.next,
2643                                                 struct ppc440spe_adma_desc_slot,
2644                                                 chain_node);
2645                         hw_desc = iter->hw_desc;
2646                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2647                         iter->src_cnt = 0;
2648                         iter->dst_cnt = 0;
2649                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2650                                                      ppc440spe_chan->pdest, 0);
2651                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest);
2652                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2653                                                       len);
2654                         iter->unmap_len = 0;
2655                         /* override pdest to preserve original P */
2656                         pdest = ppc440spe_chan->pdest;
2657                 }
2658                 if (qdest) {
2659                         struct dma_cdb *hw_desc;
2660                         struct ppc440spe_adma_chan *chan;
2661
2662                         iter = list_first_entry(&sw_desc->group_list,
2663                                                 struct ppc440spe_adma_desc_slot,
2664                                                 chain_node);
2665                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2666
2667                         if (pdest) {
2668                                 iter = list_entry(iter->chain_node.next,
2669                                                 struct ppc440spe_adma_desc_slot,
2670                                                 chain_node);
2671                         }
2672
2673                         memset(iter->hw_desc, 0, sizeof(struct dma_cdb));
2674                         iter->hw_next = list_entry(iter->chain_node.next,
2675                                                 struct ppc440spe_adma_desc_slot,
2676                                                 chain_node);
2677                         hw_desc = iter->hw_desc;
2678                         hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2;
2679                         iter->src_cnt = 0;
2680                         iter->dst_cnt = 0;
2681                         ppc440spe_desc_set_dest_addr(iter, chan, 0,
2682                                                      ppc440spe_chan->qdest, 0);
2683                         ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest);
2684                         ppc440spe_desc_set_byte_count(iter, ppc440spe_chan,
2685                                                       len);
2686                         iter->unmap_len = 0;
2687                         /* override qdest to preserve original Q */
2688                         qdest = ppc440spe_chan->qdest;
2689                 }
2690
2691                 /* Setup destinations for P/Q ops */
2692                 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest);
2693
2694                 /* Setup zero QWORDs into DCHECK CDBs */
2695                 idst = dst_cnt;
2696                 list_for_each_entry_reverse(iter, &sw_desc->group_list,
2697                                             chain_node) {
2698                         /*
2699                          * The last CDB corresponds to Q-parity check,
2700                          * the one before last CDB corresponds
2701                          * P-parity check
2702                          */
2703                         if (idst == DMA_DEST_MAX_NUM) {
2704                                 if (idst == dst_cnt) {
2705                                         set_bit(PPC440SPE_DESC_QCHECK,
2706                                                 &iter->flags);
2707                                 } else {
2708                                         set_bit(PPC440SPE_DESC_PCHECK,
2709                                                 &iter->flags);
2710                                 }
2711                         } else {
2712                                 if (qdest) {
2713                                         set_bit(PPC440SPE_DESC_QCHECK,
2714                                                 &iter->flags);
2715                                 } else {
2716                                         set_bit(PPC440SPE_DESC_PCHECK,
2717                                                 &iter->flags);
2718                                 }
2719                         }
2720                         iter->xor_check_result = pqres;
2721
2722                         /*
2723                          * set it to zero, if check fail then result will
2724                          * be updated
2725                          */
2726                         *iter->xor_check_result = 0;
2727                         ppc440spe_desc_set_dcheck(iter, ppc440spe_chan,
2728                                 ppc440spe_qword);
2729
2730                         if (!(--dst_cnt))
2731                                 break;
2732                 }
2733
2734                 /* Setup sources and mults for P/Q ops */
2735                 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list,
2736                                                      chain_node) {
2737                         struct ppc440spe_adma_chan *chan;
2738                         u32 mult_dst;
2739
2740                         chan = to_ppc440spe_adma_chan(iter->async_tx.chan);
2741                         ppc440spe_desc_set_src_addr(iter, chan, 0,
2742                                                     DMA_CUED_XOR_HB,
2743                                                     src[src_cnt - 1]);
2744                         if (qdest) {
2745                                 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 :
2746                                                            DMA_CDB_SG_DST1;
2747                                 ppc440spe_desc_set_src_mult(iter, chan,
2748                                                             DMA_CUED_MULT1_OFF,
2749                                                             mult_dst,
2750                                                             scf[src_cnt - 1]);
2751                         }
2752                         if (!(--src_cnt))
2753                                 break;
2754                 }
2755         }
2756         spin_unlock_bh(&ppc440spe_chan->lock);
2757         return sw_desc ? &sw_desc->async_tx : NULL;
2758 }
2759
2760 /**
2761  * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for
2762  * XOR ZERO_SUM operation
2763  */
2764 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum(
2765                 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt,
2766                 size_t len, enum sum_check_flags *result, unsigned long flags)
2767 {
2768         struct dma_async_tx_descriptor *tx;
2769         dma_addr_t pq[2];
2770
2771         /* validate P, disable Q */
2772         pq[0] = src[0];
2773         pq[1] = 0;
2774         flags |= DMA_PREP_PQ_DISABLE_Q;
2775
2776         tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1],
2777                                                 src_cnt - 1, 0, len,
2778                                                 result, flags);
2779         return tx;
2780 }
2781
2782 /**
2783  * ppc440spe_adma_set_dest - set destination address into descriptor
2784  */
2785 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2786                 dma_addr_t addr, int index)
2787 {
2788         struct ppc440spe_adma_chan *chan;
2789
2790         BUG_ON(index >= sw_desc->dst_cnt);
2791
2792         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2793
2794         switch (chan->device->id) {
2795         case PPC440SPE_DMA0_ID:
2796         case PPC440SPE_DMA1_ID:
2797                 /* to do: support transfers lengths >
2798                  * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT
2799                  */
2800                 ppc440spe_desc_set_dest_addr(sw_desc->group_head,
2801                         chan, 0, addr, index);
2802                 break;
2803         case PPC440SPE_XOR_ID:
2804                 sw_desc = ppc440spe_get_group_entry(sw_desc, index);
2805                 ppc440spe_desc_set_dest_addr(sw_desc,
2806                         chan, 0, addr, index);
2807                 break;
2808         }
2809 }
2810
2811 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter,
2812                 struct ppc440spe_adma_chan *chan, dma_addr_t addr)
2813 {
2814         /*  To clear destinations update the descriptor
2815          * (P or Q depending on index) as follows:
2816          * addr is destination (0 corresponds to SG2):
2817          */
2818         ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0);
2819
2820         /* ... and the addr is source: */
2821         ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr);
2822
2823         /* addr is always SG2 then the mult is always DST1 */
2824         ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF,
2825                                     DMA_CDB_SG_DST1, 1);
2826 }
2827
2828 /**
2829  * ppc440spe_adma_pq_set_dest - set destination address into descriptor
2830  * for the PQXOR operation
2831  */
2832 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc,
2833                 dma_addr_t *addrs, unsigned long flags)
2834 {
2835         struct ppc440spe_adma_desc_slot *iter;
2836         struct ppc440spe_adma_chan *chan;
2837         dma_addr_t paddr, qaddr;
2838         dma_addr_t addr = 0, ppath, qpath;
2839         int index = 0, i;
2840
2841         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
2842
2843         if (flags & DMA_PREP_PQ_DISABLE_P)
2844                 paddr = 0;
2845         else
2846                 paddr = addrs[0];
2847
2848         if (flags & DMA_PREP_PQ_DISABLE_Q)
2849                 qaddr = 0;
2850         else
2851                 qaddr = addrs[1];
2852
2853         if (!paddr || !qaddr)
2854                 addr = paddr ? paddr : qaddr;
2855
2856         switch (chan->device->id) {
2857         case PPC440SPE_DMA0_ID:
2858         case PPC440SPE_DMA1_ID:
2859                 /* walk through the WXOR source list and set P/Q-destinations
2860                  * for each slot:
2861                  */
2862                 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
2863                         /* This is WXOR-only chain; may have 1/2 zero descs */
2864                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
2865                                 index++;
2866                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
2867                                 index++;
2868
2869                         iter = ppc440spe_get_group_entry(sw_desc, index);
2870                         if (addr) {
2871                                 /* one destination */
2872                                 list_for_each_entry_from(iter,
2873                                         &sw_desc->group_list, chain_node)
2874                                         ppc440spe_desc_set_dest_addr(iter, chan,
2875                                                 DMA_CUED_XOR_BASE, addr, 0);
2876                         } else {
2877                                 /* two destinations */
2878                                 list_for_each_entry_from(iter,
2879                                         &sw_desc->group_list, chain_node) {
2880                                         ppc440spe_desc_set_dest_addr(iter, chan,
2881                                                 DMA_CUED_XOR_BASE, paddr, 0);
2882                                         ppc440spe_desc_set_dest_addr(iter, chan,
2883                                                 DMA_CUED_XOR_BASE, qaddr, 1);
2884                                 }
2885                         }
2886
2887                         if (index) {
2888                                 /*  To clear destinations update the descriptor
2889                                  * (1st,2nd, or both depending on flags)
2890                                  */
2891                                 index = 0;
2892                                 if (test_bit(PPC440SPE_ZERO_P,
2893                                                 &sw_desc->flags)) {
2894                                         iter = ppc440spe_get_group_entry(
2895                                                         sw_desc, index++);
2896                                         ppc440spe_adma_pq_zero_op(iter, chan,
2897                                                         paddr);
2898                                 }
2899
2900                                 if (test_bit(PPC440SPE_ZERO_Q,
2901                                                 &sw_desc->flags)) {
2902                                         iter = ppc440spe_get_group_entry(
2903                                                         sw_desc, index++);
2904                                         ppc440spe_adma_pq_zero_op(iter, chan,
2905                                                         qaddr);
2906                                 }
2907
2908                                 return;
2909                         }
2910                 } else {
2911                         /* This is RXOR-only or RXOR/WXOR mixed chain */
2912
2913                         /* If we want to include destination into calculations,
2914                          * then make dest addresses cued with mult=1 (XOR).
2915                          */
2916                         ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2917                                         DMA_CUED_XOR_HB :
2918                                         DMA_CUED_XOR_BASE |
2919                                                 (1 << DMA_CUED_MULT1_OFF);
2920                         qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2921                                         DMA_CUED_XOR_HB :
2922                                         DMA_CUED_XOR_BASE |
2923                                                 (1 << DMA_CUED_MULT1_OFF);
2924
2925                         /* Setup destination(s) in RXOR slot(s) */
2926                         iter = ppc440spe_get_group_entry(sw_desc, index++);
2927                         ppc440spe_desc_set_dest_addr(iter, chan,
2928                                                 paddr ? ppath : qpath,
2929                                                 paddr ? paddr : qaddr, 0);
2930                         if (!addr) {
2931                                 /* two destinations */
2932                                 iter = ppc440spe_get_group_entry(sw_desc,
2933                                                                  index++);
2934                                 ppc440spe_desc_set_dest_addr(iter, chan,
2935                                                 qpath, qaddr, 0);
2936                         }
2937
2938                         if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) {
2939                                 /* Setup destination(s) in remaining WXOR
2940                                  * slots
2941                                  */
2942                                 iter = ppc440spe_get_group_entry(sw_desc,
2943                                                                  index);
2944                                 if (addr) {
2945                                         /* one destination */
2946                                         list_for_each_entry_from(iter,
2947                                             &sw_desc->group_list,
2948                                             chain_node)
2949                                                 ppc440spe_desc_set_dest_addr(
2950                                                         iter, chan,
2951                                                         DMA_CUED_XOR_BASE,
2952                                                         addr, 0);
2953
2954                                 } else {
2955                                         /* two destinations */
2956                                         list_for_each_entry_from(iter,
2957                                             &sw_desc->group_list,
2958                                             chain_node) {
2959                                                 ppc440spe_desc_set_dest_addr(
2960                                                         iter, chan,
2961                                                         DMA_CUED_XOR_BASE,
2962                                                         paddr, 0);
2963                                                 ppc440spe_desc_set_dest_addr(
2964                                                         iter, chan,
2965                                                         DMA_CUED_XOR_BASE,
2966                                                         qaddr, 1);
2967                                         }
2968                                 }
2969                         }
2970
2971                 }
2972                 break;
2973
2974         case PPC440SPE_XOR_ID:
2975                 /* DMA2 descriptors have only 1 destination, so there are
2976                  * two chains - one for each dest.
2977                  * If we want to include destination into calculations,
2978                  * then make dest addresses cued with mult=1 (XOR).
2979                  */
2980                 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ?
2981                                 DMA_CUED_XOR_HB :
2982                                 DMA_CUED_XOR_BASE |
2983                                         (1 << DMA_CUED_MULT1_OFF);
2984
2985                 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ?
2986                                 DMA_CUED_XOR_HB :
2987                                 DMA_CUED_XOR_BASE |
2988                                         (1 << DMA_CUED_MULT1_OFF);
2989
2990                 iter = ppc440spe_get_group_entry(sw_desc, 0);
2991                 for (i = 0; i < sw_desc->descs_per_op; i++) {
2992                         ppc440spe_desc_set_dest_addr(iter, chan,
2993                                 paddr ? ppath : qpath,
2994                                 paddr ? paddr : qaddr, 0);
2995                         iter = list_entry(iter->chain_node.next,
2996                                           struct ppc440spe_adma_desc_slot,
2997                                           chain_node);
2998                 }
2999
3000                 if (!addr) {
3001                         /* Two destinations; setup Q here */
3002                         iter = ppc440spe_get_group_entry(sw_desc,
3003                                 sw_desc->descs_per_op);
3004                         for (i = 0; i < sw_desc->descs_per_op; i++) {
3005                                 ppc440spe_desc_set_dest_addr(iter,
3006                                         chan, qpath, qaddr, 0);
3007                                 iter = list_entry(iter->chain_node.next,
3008                                                 struct ppc440spe_adma_desc_slot,
3009                                                 chain_node);
3010                         }
3011                 }
3012
3013                 break;
3014         }
3015 }
3016
3017 /**
3018  * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor
3019  * for the PQ_ZERO_SUM operation
3020  */
3021 static void ppc440spe_adma_pqzero_sum_set_dest(
3022                 struct ppc440spe_adma_desc_slot *sw_desc,
3023                 dma_addr_t paddr, dma_addr_t qaddr)
3024 {
3025         struct ppc440spe_adma_desc_slot *iter, *end;
3026         struct ppc440spe_adma_chan *chan;
3027         dma_addr_t addr = 0;
3028         int idx;
3029
3030         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3031
3032         /* walk through the WXOR source list and set P/Q-destinations
3033          * for each slot
3034          */
3035         idx = (paddr && qaddr) ? 2 : 1;
3036         /* set end */
3037         list_for_each_entry_reverse(end, &sw_desc->group_list,
3038                                     chain_node) {
3039                 if (!(--idx))
3040                         break;
3041         }
3042         /* set start */
3043         idx = (paddr && qaddr) ? 2 : 1;
3044         iter = ppc440spe_get_group_entry(sw_desc, idx);
3045
3046         if (paddr && qaddr) {
3047                 /* two destinations */
3048                 list_for_each_entry_from(iter, &sw_desc->group_list,
3049                                          chain_node) {
3050                         if (unlikely(iter == end))
3051                                 break;
3052                         ppc440spe_desc_set_dest_addr(iter, chan,
3053                                                 DMA_CUED_XOR_BASE, paddr, 0);
3054                         ppc440spe_desc_set_dest_addr(iter, chan,
3055                                                 DMA_CUED_XOR_BASE, qaddr, 1);
3056                 }
3057         } else {
3058                 /* one destination */
3059                 addr = paddr ? paddr : qaddr;
3060                 list_for_each_entry_from(iter, &sw_desc->group_list,
3061                                          chain_node) {
3062                         if (unlikely(iter == end))
3063                                 break;
3064                         ppc440spe_desc_set_dest_addr(iter, chan,
3065                                                 DMA_CUED_XOR_BASE, addr, 0);
3066                 }
3067         }
3068
3069         /*  The remaining descriptors are DATACHECK. These have no need in
3070          * destination. Actually, these destinations are used there
3071          * as sources for check operation. So, set addr as source.
3072          */
3073         ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr);
3074
3075         if (!addr) {
3076                 end = list_entry(end->chain_node.next,
3077                                  struct ppc440spe_adma_desc_slot, chain_node);
3078                 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr);
3079         }
3080 }
3081
3082 /**
3083  * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor
3084  */
3085 static inline void ppc440spe_desc_set_xor_src_cnt(
3086                         struct ppc440spe_adma_desc_slot *desc,
3087                         int src_cnt)
3088 {
3089         struct xor_cb *hw_desc = desc->hw_desc;
3090
3091         hw_desc->cbc &= ~XOR_CDCR_OAC_MSK;
3092         hw_desc->cbc |= src_cnt;
3093 }
3094
3095 /**
3096  * ppc440spe_adma_pq_set_src - set source address into descriptor
3097  */
3098 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc,
3099                 dma_addr_t addr, int index)
3100 {
3101         struct ppc440spe_adma_chan *chan;
3102         dma_addr_t haddr = 0;
3103         struct ppc440spe_adma_desc_slot *iter = NULL;
3104
3105         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3106
3107         switch (chan->device->id) {
3108         case PPC440SPE_DMA0_ID:
3109         case PPC440SPE_DMA1_ID:
3110                 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain
3111                  */
3112                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3113                         /* RXOR-only or RXOR/WXOR operation */
3114                         int iskip = test_bit(PPC440SPE_DESC_RXOR12,
3115                                 &sw_desc->flags) ?  2 : 3;
3116
3117                         if (index == 0) {
3118                                 /* 1st slot (RXOR) */
3119                                 /* setup sources region (R1-2-3, R1-2-4,
3120                                  * or R1-2-5)
3121                                  */
3122                                 if (test_bit(PPC440SPE_DESC_RXOR12,
3123                                                 &sw_desc->flags))
3124                                         haddr = DMA_RXOR12 <<
3125                                                 DMA_CUED_REGION_OFF;
3126                                 else if (test_bit(PPC440SPE_DESC_RXOR123,
3127                                     &sw_desc->flags))
3128                                         haddr = DMA_RXOR123 <<
3129                                                 DMA_CUED_REGION_OFF;
3130                                 else if (test_bit(PPC440SPE_DESC_RXOR124,
3131                                     &sw_desc->flags))
3132                                         haddr = DMA_RXOR124 <<
3133                                                 DMA_CUED_REGION_OFF;
3134                                 else if (test_bit(PPC440SPE_DESC_RXOR125,
3135                                     &sw_desc->flags))
3136                                         haddr = DMA_RXOR125 <<
3137                                                 DMA_CUED_REGION_OFF;
3138                                 else
3139                                         BUG();
3140                                 haddr |= DMA_CUED_XOR_BASE;
3141                                 iter = ppc440spe_get_group_entry(sw_desc, 0);
3142                         } else if (index < iskip) {
3143                                 /* 1st slot (RXOR)
3144                                  * shall actually set source address only once
3145                                  * instead of first <iskip>
3146                                  */
3147                                 iter = NULL;
3148                         } else {
3149                                 /* 2nd/3d and next slots (WXOR);
3150                                  * skip first slot with RXOR
3151                                  */
3152                                 haddr = DMA_CUED_XOR_HB;
3153                                 iter = ppc440spe_get_group_entry(sw_desc,
3154                                     index - iskip + sw_desc->dst_cnt);
3155                         }
3156                 } else {
3157                         int znum = 0;
3158
3159                         /* WXOR-only operation; skip first slots with
3160                          * zeroing destinations
3161                          */
3162                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3163                                 znum++;
3164                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3165                                 znum++;
3166
3167                         haddr = DMA_CUED_XOR_HB;
3168                         iter = ppc440spe_get_group_entry(sw_desc,
3169                                         index + znum);
3170                 }
3171
3172                 if (likely(iter)) {
3173                         ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr);
3174
3175                         if (!index &&
3176                             test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) &&
3177                             sw_desc->dst_cnt == 2) {
3178                                 /* if we have two destinations for RXOR, then
3179                                  * setup source in the second descr too
3180                                  */
3181                                 iter = ppc440spe_get_group_entry(sw_desc, 1);
3182                                 ppc440spe_desc_set_src_addr(iter, chan, 0,
3183                                         haddr, addr);
3184                         }
3185                 }
3186                 break;
3187
3188         case PPC440SPE_XOR_ID:
3189                 /* DMA2 may do Biskup */
3190                 iter = sw_desc->group_head;
3191                 if (iter->dst_cnt == 2) {
3192                         /* both P & Q calculations required; set P src here */
3193                         ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3194
3195                         /* this is for Q */
3196                         iter = ppc440spe_get_group_entry(sw_desc,
3197                                 sw_desc->descs_per_op);
3198                 }
3199                 ppc440spe_adma_dma2rxor_set_src(iter, index, addr);
3200                 break;
3201         }
3202 }
3203
3204 /**
3205  * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor
3206  */
3207 static void ppc440spe_adma_memcpy_xor_set_src(
3208                 struct ppc440spe_adma_desc_slot *sw_desc,
3209                 dma_addr_t addr, int index)
3210 {
3211         struct ppc440spe_adma_chan *chan;
3212
3213         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3214         sw_desc = sw_desc->group_head;
3215
3216         if (likely(sw_desc))
3217                 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr);
3218 }
3219
3220 /**
3221  * ppc440spe_adma_dma2rxor_inc_addr  -
3222  */
3223 static void ppc440spe_adma_dma2rxor_inc_addr(
3224                 struct ppc440spe_adma_desc_slot *desc,
3225                 struct ppc440spe_rxor *cursor, int index, int src_cnt)
3226 {
3227         cursor->addr_count++;
3228         if (index == src_cnt - 1) {
3229                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3230         } else if (cursor->addr_count == XOR_MAX_OPS) {
3231                 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count);
3232                 cursor->addr_count = 0;
3233                 cursor->desc_count++;
3234         }
3235 }
3236
3237 /**
3238  * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB
3239  */
3240 static int ppc440spe_adma_dma2rxor_prep_src(
3241                 struct ppc440spe_adma_desc_slot *hdesc,
3242                 struct ppc440spe_rxor *cursor, int index,
3243                 int src_cnt, u32 addr)
3244 {
3245         int rval = 0;
3246         u32 sign;
3247         struct ppc440spe_adma_desc_slot *desc = hdesc;
3248         int i;
3249
3250         for (i = 0; i < cursor->desc_count; i++) {
3251                 desc = list_entry(hdesc->chain_node.next,
3252                                   struct ppc440spe_adma_desc_slot,
3253                                   chain_node);
3254         }
3255
3256         switch (cursor->state) {
3257         case 0:
3258                 if (addr == cursor->addrl + cursor->len) {
3259                         /* direct RXOR */
3260                         cursor->state = 1;
3261                         cursor->xor_count++;
3262                         if (index == src_cnt-1) {
3263                                 ppc440spe_rxor_set_region(desc,
3264                                         cursor->addr_count,
3265                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3266                                 ppc440spe_adma_dma2rxor_inc_addr(
3267                                         desc, cursor, index, src_cnt);
3268                         }
3269                 } else if (cursor->addrl == addr + cursor->len) {
3270                         /* reverse RXOR */
3271                         cursor->state = 1;
3272                         cursor->xor_count++;
3273                         set_bit(cursor->addr_count, &desc->reverse_flags[0]);
3274                         if (index == src_cnt-1) {
3275                                 ppc440spe_rxor_set_region(desc,
3276                                         cursor->addr_count,
3277                                         DMA_RXOR12 << DMA_CUED_REGION_OFF);
3278                                 ppc440spe_adma_dma2rxor_inc_addr(
3279                                         desc, cursor, index, src_cnt);
3280                         }
3281                 } else {
3282                         printk(KERN_ERR "Cannot build "
3283                                 "DMA2 RXOR command block.\n");
3284                         BUG();
3285                 }
3286                 break;
3287         case 1:
3288                 sign = test_bit(cursor->addr_count,
3289                                 desc->reverse_flags)
3290                         ? -1 : 1;
3291                 if (index == src_cnt-2 || (sign == -1
3292                         && addr != cursor->addrl - 2*cursor->len)) {
3293                         cursor->state = 0;
3294                         cursor->xor_count = 1;
3295                         cursor->addrl = addr;
3296                         ppc440spe_rxor_set_region(desc,
3297                                 cursor->addr_count,
3298                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3299                         ppc440spe_adma_dma2rxor_inc_addr(
3300                                 desc, cursor, index, src_cnt);
3301                 } else if (addr == cursor->addrl + 2*sign*cursor->len) {
3302                         cursor->state = 2;
3303                         cursor->xor_count = 0;
3304                         ppc440spe_rxor_set_region(desc,
3305                                 cursor->addr_count,
3306                                 DMA_RXOR123 << DMA_CUED_REGION_OFF);
3307                         if (index == src_cnt-1) {
3308                                 ppc440spe_adma_dma2rxor_inc_addr(
3309                                         desc, cursor, index, src_cnt);
3310                         }
3311                 } else if (addr == cursor->addrl + 3*cursor->len) {
3312                         cursor->state = 2;
3313                         cursor->xor_count = 0;
3314                         ppc440spe_rxor_set_region(desc,
3315                                 cursor->addr_count,
3316                                 DMA_RXOR124 << DMA_CUED_REGION_OFF);
3317                         if (index == src_cnt-1) {
3318                                 ppc440spe_adma_dma2rxor_inc_addr(
3319                                         desc, cursor, index, src_cnt);
3320                         }
3321                 } else if (addr == cursor->addrl + 4*cursor->len) {
3322                         cursor->state = 2;
3323                         cursor->xor_count = 0;
3324                         ppc440spe_rxor_set_region(desc,
3325                                 cursor->addr_count,
3326                                 DMA_RXOR125 << DMA_CUED_REGION_OFF);
3327                         if (index == src_cnt-1) {
3328                                 ppc440spe_adma_dma2rxor_inc_addr(
3329                                         desc, cursor, index, src_cnt);
3330                         }
3331                 } else {
3332                         cursor->state = 0;
3333                         cursor->xor_count = 1;
3334                         cursor->addrl = addr;
3335                         ppc440spe_rxor_set_region(desc,
3336                                 cursor->addr_count,
3337                                 DMA_RXOR12 << DMA_CUED_REGION_OFF);
3338                         ppc440spe_adma_dma2rxor_inc_addr(
3339                                 desc, cursor, index, src_cnt);
3340                 }
3341                 break;
3342         case 2:
3343                 cursor->state = 0;
3344                 cursor->addrl = addr;
3345                 cursor->xor_count++;
3346                 if (index) {
3347                         ppc440spe_adma_dma2rxor_inc_addr(
3348                                 desc, cursor, index, src_cnt);
3349                 }
3350                 break;
3351         }
3352
3353         return rval;
3354 }
3355
3356 /**
3357  * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that
3358  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3359  */
3360 static void ppc440spe_adma_dma2rxor_set_src(
3361                 struct ppc440spe_adma_desc_slot *desc,
3362                 int index, dma_addr_t addr)
3363 {
3364         struct xor_cb *xcb = desc->hw_desc;
3365         int k = 0, op = 0, lop = 0;
3366
3367         /* get the RXOR operand which corresponds to index addr */
3368         while (op <= index) {
3369                 lop = op;
3370                 if (k == XOR_MAX_OPS) {
3371                         k = 0;
3372                         desc = list_entry(desc->chain_node.next,
3373                                 struct ppc440spe_adma_desc_slot, chain_node);
3374                         xcb = desc->hw_desc;
3375
3376                 }
3377                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3378                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3379                         op += 2;
3380                 else
3381                         op += 3;
3382         }
3383
3384         BUG_ON(k < 1);
3385
3386         if (test_bit(k-1, desc->reverse_flags)) {
3387                 /* reverse operand order; put last op in RXOR group */
3388                 if (index == op - 1)
3389                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3390         } else {
3391                 /* direct operand order; put first op in RXOR group */
3392                 if (index == lop)
3393                         ppc440spe_rxor_set_src(desc, k - 1, addr);
3394         }
3395 }
3396
3397 /**
3398  * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that
3399  *      ppc440spe_adma_dma2rxor_prep_src() has already done prior this call
3400  */
3401 static void ppc440spe_adma_dma2rxor_set_mult(
3402                 struct ppc440spe_adma_desc_slot *desc,
3403                 int index, u8 mult)
3404 {
3405         struct xor_cb *xcb = desc->hw_desc;
3406         int k = 0, op = 0, lop = 0;
3407
3408         /* get the RXOR operand which corresponds to index mult */
3409         while (op <= index) {
3410                 lop = op;
3411                 if (k == XOR_MAX_OPS) {
3412                         k = 0;
3413                         desc = list_entry(desc->chain_node.next,
3414                                           struct ppc440spe_adma_desc_slot,
3415                                           chain_node);
3416                         xcb = desc->hw_desc;
3417
3418                 }
3419                 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) ==
3420                     (DMA_RXOR12 << DMA_CUED_REGION_OFF))
3421                         op += 2;
3422                 else
3423                         op += 3;
3424         }
3425
3426         BUG_ON(k < 1);
3427         if (test_bit(k-1, desc->reverse_flags)) {
3428                 /* reverse order */
3429                 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult);
3430         } else {
3431                 /* direct order */
3432                 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult);
3433         }
3434 }
3435
3436 /**
3437  * ppc440spe_init_rxor_cursor -
3438  */
3439 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor)
3440 {
3441         memset(cursor, 0, sizeof(struct ppc440spe_rxor));
3442         cursor->state = 2;
3443 }
3444
3445 /**
3446  * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into
3447  * descriptor for the PQXOR operation
3448  */
3449 static void ppc440spe_adma_pq_set_src_mult(
3450                 struct ppc440spe_adma_desc_slot *sw_desc,
3451                 unsigned char mult, int index, int dst_pos)
3452 {
3453         struct ppc440spe_adma_chan *chan;
3454         u32 mult_idx, mult_dst;
3455         struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL;
3456
3457         chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan);
3458
3459         switch (chan->device->id) {
3460         case PPC440SPE_DMA0_ID:
3461         case PPC440SPE_DMA1_ID:
3462                 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) {
3463                         int region = test_bit(PPC440SPE_DESC_RXOR12,
3464                                         &sw_desc->flags) ? 2 : 3;
3465
3466                         if (index < region) {
3467                                 /* RXOR multipliers */
3468                                 iter = ppc440spe_get_group_entry(sw_desc,
3469                                         sw_desc->dst_cnt - 1);
3470                                 if (sw_desc->dst_cnt == 2)
3471                                         iter1 = ppc440spe_get_group_entry(
3472                                                         sw_desc, 0);
3473
3474                                 mult_idx = DMA_CUED_MULT1_OFF + (index << 3);
3475                                 mult_dst = DMA_CDB_SG_SRC;
3476                         } else {
3477                                 /* WXOR multiplier */
3478                                 iter = ppc440spe_get_group_entry(sw_desc,
3479                                                         index - region +
3480                                                         sw_desc->dst_cnt);
3481                                 mult_idx = DMA_CUED_MULT1_OFF;
3482                                 mult_dst = dst_pos ? DMA_CDB_SG_DST2 :
3483                                                      DMA_CDB_SG_DST1;
3484                         }
3485                 } else {
3486                         int znum = 0;
3487
3488                         /* WXOR-only;
3489                          * skip first slots with destinations (if ZERO_DST has
3490                          * place)
3491                          */
3492                         if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags))
3493                                 znum++;
3494                         if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags))
3495                                 znum++;
3496
3497                         iter = ppc440spe_get_group_entry(sw_desc, index + znum);
3498                         mult_idx = DMA_CUED_MULT1_OFF;
3499                         mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1;
3500                 }
3501
3502                 if (likely(iter)) {
3503                         ppc440spe_desc_set_src_mult(iter, chan,
3504                                 mult_idx, mult_dst, mult);
3505
3506                         if (unlikely(iter1)) {
3507                                 /* if we have two destinations for RXOR, then
3508                                  * we've just set Q mult. Set-up P now.
3509                                  */
3510                                 ppc440spe_desc_set_src_mult(iter1, chan,
3511                                         mult_idx, mult_dst, 1);
3512                         }
3513
3514                 }
3515                 break;
3516
3517         case PPC440SPE_XOR_ID:
3518                 iter = sw_desc->group_head;
3519                 if (sw_desc->dst_cnt == 2) {
3520                         /* both P & Q calculations required; set P mult here */
3521                         ppc440spe_adma_dma2rxor_set_mult(iter, index, 1);
3522
3523                         /* and then set Q mult */
3524                         iter = ppc440spe_get_group_entry(sw_desc,
3525                                sw_desc->descs_per_op);
3526                 }
3527                 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult);
3528                 break;
3529         }
3530 }
3531
3532 /**
3533  * ppc440spe_adma_free_chan_resources - free the resources allocated
3534  */
3535 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan)
3536 {
3537         struct ppc440spe_adma_chan *ppc440spe_chan;
3538         struct ppc440spe_adma_desc_slot *iter, *_iter;
3539         int in_use_descs = 0;
3540
3541         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3542         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3543
3544         spin_lock_bh(&ppc440spe_chan->lock);
3545         list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain,
3546                                         chain_node) {
3547                 in_use_descs++;
3548                 list_del(&iter->chain_node);
3549         }
3550         list_for_each_entry_safe_reverse(iter, _iter,
3551                         &ppc440spe_chan->all_slots, slot_node) {
3552                 list_del(&iter->slot_node);
3553                 kfree(iter);
3554                 ppc440spe_chan->slots_allocated--;
3555         }
3556         ppc440spe_chan->last_used = NULL;
3557
3558         dev_dbg(ppc440spe_chan->device->common.dev,
3559                 "ppc440spe adma%d %s slots_allocated %d\n",
3560                 ppc440spe_chan->device->id,
3561                 __func__, ppc440spe_chan->slots_allocated);
3562         spin_unlock_bh(&ppc440spe_chan->lock);
3563
3564         /* one is ok since we left it on there on purpose */
3565         if (in_use_descs > 1)
3566                 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n",
3567                         in_use_descs - 1);
3568 }
3569
3570 /**
3571  * ppc440spe_adma_tx_status - poll the status of an ADMA transaction
3572  * @chan: ADMA channel handle
3573  * @cookie: ADMA transaction identifier
3574  * @txstate: a holder for the current state of the channel
3575  */
3576 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan,
3577                         dma_cookie_t cookie, struct dma_tx_state *txstate)
3578 {
3579         struct ppc440spe_adma_chan *ppc440spe_chan;
3580         enum dma_status ret;
3581
3582         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3583         ret = dma_cookie_status(chan, cookie, txstate);
3584         if (ret == DMA_COMPLETE)
3585                 return ret;
3586
3587         ppc440spe_adma_slot_cleanup(ppc440spe_chan);
3588
3589         return dma_cookie_status(chan, cookie, txstate);
3590 }
3591
3592 /**
3593  * ppc440spe_adma_eot_handler - end of transfer interrupt handler
3594  */
3595 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data)
3596 {
3597         struct ppc440spe_adma_chan *chan = data;
3598
3599         dev_dbg(chan->device->common.dev,
3600                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3601
3602         tasklet_schedule(&chan->irq_tasklet);
3603         ppc440spe_adma_device_clear_eot_status(chan);
3604
3605         return IRQ_HANDLED;
3606 }
3607
3608 /**
3609  * ppc440spe_adma_err_handler - DMA error interrupt handler;
3610  *      do the same things as a eot handler
3611  */
3612 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data)
3613 {
3614         struct ppc440spe_adma_chan *chan = data;
3615
3616         dev_dbg(chan->device->common.dev,
3617                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3618
3619         tasklet_schedule(&chan->irq_tasklet);
3620         ppc440spe_adma_device_clear_eot_status(chan);
3621
3622         return IRQ_HANDLED;
3623 }
3624
3625 /**
3626  * ppc440spe_test_callback - called when test operation has been done
3627  */
3628 static void ppc440spe_test_callback(void *unused)
3629 {
3630         complete(&ppc440spe_r6_test_comp);
3631 }
3632
3633 /**
3634  * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w
3635  */
3636 static void ppc440spe_adma_issue_pending(struct dma_chan *chan)
3637 {
3638         struct ppc440spe_adma_chan *ppc440spe_chan;
3639
3640         ppc440spe_chan = to_ppc440spe_adma_chan(chan);
3641         dev_dbg(ppc440spe_chan->device->common.dev,
3642                 "ppc440spe adma%d: %s %d \n", ppc440spe_chan->device->id,
3643                 __func__, ppc440spe_chan->pending);
3644
3645         if (ppc440spe_chan->pending) {
3646                 ppc440spe_chan->pending = 0;
3647                 ppc440spe_chan_append(ppc440spe_chan);
3648         }
3649 }
3650
3651 /**
3652  * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines
3653  *      use FIFOs (as opposite to chains used in XOR) so this is a XOR
3654  *      specific operation)
3655  */
3656 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan)
3657 {
3658         struct ppc440spe_adma_desc_slot *sw_desc, *group_start;
3659         dma_cookie_t cookie;
3660         int slot_cnt, slots_per_op;
3661
3662         dev_dbg(chan->device->common.dev,
3663                 "ppc440spe adma%d: %s\n", chan->device->id, __func__);
3664
3665         spin_lock_bh(&chan->lock);
3666         slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op);
3667         sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op);
3668         if (sw_desc) {
3669                 group_start = sw_desc->group_head;
3670                 list_splice_init(&sw_desc->group_list, &chan->chain);
3671                 async_tx_ack(&sw_desc->async_tx);
3672                 ppc440spe_desc_init_null_xor(group_start);
3673
3674                 cookie = dma_cookie_assign(&sw_desc->async_tx);
3675
3676                 /* initialize the completed cookie to be less than
3677                  * the most recently used cookie
3678                  */
3679                 chan->common.completed_cookie = cookie - 1;
3680
3681                 /* channel should not be busy */
3682                 BUG_ON(ppc440spe_chan_is_busy(chan));
3683
3684                 /* set the descriptor address */
3685                 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc);
3686
3687                 /* run the descriptor */
3688                 ppc440spe_chan_run(chan);
3689         } else
3690                 printk(KERN_ERR "ppc440spe adma%d"
3691                         " failed to allocate null descriptor\n",
3692                         chan->device->id);
3693         spin_unlock_bh(&chan->lock);
3694 }
3695
3696 /**
3697  * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully.
3698  *      For this we just perform one WXOR operation with the same source
3699  *      and destination addresses, the GF-multiplier is 1; so if RAID-6
3700  *      capabilities are enabled then we'll get src/dst filled with zero.
3701  */
3702 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan)
3703 {
3704         struct ppc440spe_adma_desc_slot *sw_desc, *iter;
3705         struct page *pg;
3706         char *a;
3707         dma_addr_t dma_addr, addrs[2];
3708         unsigned long op = 0;
3709         int rval = 0;
3710
3711         set_bit(PPC440SPE_DESC_WXOR, &op);
3712
3713         pg = alloc_page(GFP_KERNEL);
3714         if (!pg)
3715                 return -ENOMEM;
3716
3717         spin_lock_bh(&chan->lock);
3718         sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1);
3719         if (sw_desc) {
3720                 /* 1 src, 1 dsr, int_ena, WXOR */
3721                 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op);
3722                 list_for_each_entry(iter, &sw_desc->group_list, chain_node) {
3723                         ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE);
3724                         iter->unmap_len = PAGE_SIZE;
3725                 }
3726         } else {
3727                 rval = -EFAULT;
3728                 spin_unlock_bh(&chan->lock);
3729                 goto exit;
3730         }
3731         spin_unlock_bh(&chan->lock);
3732
3733         /* Fill the test page with ones */
3734         memset(page_address(pg), 0xFF, PAGE_SIZE);
3735         dma_addr = dma_map_page(chan->device->dev, pg, 0,
3736                                 PAGE_SIZE, DMA_BIDIRECTIONAL);
3737
3738         /* Setup addresses */
3739         ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0);
3740         ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0);
3741         addrs[0] = dma_addr;
3742         addrs[1] = 0;
3743         ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q);
3744
3745         async_tx_ack(&sw_desc->async_tx);
3746         sw_desc->async_tx.callback = ppc440spe_test_callback;
3747         sw_desc->async_tx.callback_param = NULL;
3748
3749         init_completion(&ppc440spe_r6_test_comp);
3750
3751         ppc440spe_adma_tx_submit(&sw_desc->async_tx);
3752         ppc440spe_adma_issue_pending(&chan->common);
3753
3754         wait_for_completion(&ppc440spe_r6_test_comp);
3755
3756         /* Now check if the test page is zeroed */
3757         a = page_address(pg);
3758         if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) {
3759                 /* page is zero - RAID-6 enabled */
3760                 rval = 0;
3761         } else {
3762                 /* RAID-6 was not enabled */
3763                 rval = -EINVAL;
3764         }
3765 exit:
3766         __free_page(pg);
3767         return rval;
3768 }
3769
3770 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev)
3771 {
3772         switch (adev->id) {
3773         case PPC440SPE_DMA0_ID:
3774         case PPC440SPE_DMA1_ID:
3775                 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask);
3776                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3777                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3778                 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask);
3779                 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask);
3780                 break;
3781         case PPC440SPE_XOR_ID:
3782                 dma_cap_set(DMA_XOR, adev->common.cap_mask);
3783                 dma_cap_set(DMA_PQ, adev->common.cap_mask);
3784                 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask);
3785                 adev->common.cap_mask = adev->common.cap_mask;
3786                 break;
3787         }
3788
3789         /* Set base routines */
3790         adev->common.device_alloc_chan_resources =
3791                                 ppc440spe_adma_alloc_chan_resources;
3792         adev->common.device_free_chan_resources =
3793                                 ppc440spe_adma_free_chan_resources;
3794         adev->common.device_tx_status = ppc440spe_adma_tx_status;
3795         adev->common.device_issue_pending = ppc440spe_adma_issue_pending;
3796
3797         /* Set prep routines based on capability */
3798         if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) {
3799                 adev->common.device_prep_dma_memcpy =
3800                         ppc440spe_adma_prep_dma_memcpy;
3801         }
3802         if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) {
3803                 adev->common.max_xor = XOR_MAX_OPS;
3804                 adev->common.device_prep_dma_xor =
3805                         ppc440spe_adma_prep_dma_xor;
3806         }
3807         if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) {
3808                 switch (adev->id) {
3809                 case PPC440SPE_DMA0_ID:
3810                         dma_set_maxpq(&adev->common,
3811                                 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3812                         break;
3813                 case PPC440SPE_DMA1_ID:
3814                         dma_set_maxpq(&adev->common,
3815                                 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0);
3816                         break;
3817                 case PPC440SPE_XOR_ID:
3818                         adev->common.max_pq = XOR_MAX_OPS * 3;
3819                         break;
3820                 }
3821                 adev->common.device_prep_dma_pq =
3822                         ppc440spe_adma_prep_dma_pq;
3823         }
3824         if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) {
3825                 switch (adev->id) {
3826                 case PPC440SPE_DMA0_ID:
3827                         adev->common.max_pq = DMA0_FIFO_SIZE /
3828                                                 sizeof(struct dma_cdb);
3829                         break;
3830                 case PPC440SPE_DMA1_ID:
3831                         adev->common.max_pq = DMA1_FIFO_SIZE /
3832                                                 sizeof(struct dma_cdb);
3833                         break;
3834                 }
3835                 adev->common.device_prep_dma_pq_val =
3836                         ppc440spe_adma_prep_dma_pqzero_sum;
3837         }
3838         if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) {
3839                 switch (adev->id) {
3840                 case PPC440SPE_DMA0_ID:
3841                         adev->common.max_xor = DMA0_FIFO_SIZE /
3842                                                 sizeof(struct dma_cdb);
3843                         break;
3844                 case PPC440SPE_DMA1_ID:
3845                         adev->common.max_xor = DMA1_FIFO_SIZE /
3846                                                 sizeof(struct dma_cdb);
3847                         break;
3848                 }
3849                 adev->common.device_prep_dma_xor_val =
3850                         ppc440spe_adma_prep_dma_xor_zero_sum;
3851         }
3852         if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) {
3853                 adev->common.device_prep_dma_interrupt =
3854                         ppc440spe_adma_prep_dma_interrupt;
3855         }
3856         pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: "
3857           "( %s%s%s%s%s%s)\n",
3858           dev_name(adev->dev),
3859           dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "",
3860           dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "",
3861           dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "",
3862           dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "",
3863           dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "",
3864           dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : "");
3865 }
3866
3867 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev,
3868                                      struct ppc440spe_adma_chan *chan,
3869                                      int *initcode)
3870 {
3871         struct platform_device *ofdev;
3872         struct device_node *np;
3873         int ret;
3874
3875         ofdev = container_of(adev->dev, struct platform_device, dev);
3876         np = ofdev->dev.of_node;
3877         if (adev->id != PPC440SPE_XOR_ID) {
3878                 adev->err_irq = irq_of_parse_and_map(np, 1);
3879                 if (!adev->err_irq) {
3880                         dev_warn(adev->dev, "no err irq resource?\n");
3881                         *initcode = PPC_ADMA_INIT_IRQ2;
3882                         adev->err_irq = -ENXIO;
3883                 } else
3884                         atomic_inc(&ppc440spe_adma_err_irq_ref);
3885         } else {
3886                 adev->err_irq = -ENXIO;
3887         }
3888
3889         adev->irq = irq_of_parse_and_map(np, 0);
3890         if (!adev->irq) {
3891                 dev_err(adev->dev, "no irq resource\n");
3892                 *initcode = PPC_ADMA_INIT_IRQ1;
3893                 ret = -ENXIO;
3894                 goto err_irq_map;
3895         }
3896         dev_dbg(adev->dev, "irq %d, err irq %d\n",
3897                 adev->irq, adev->err_irq);
3898
3899         ret = request_irq(adev->irq, ppc440spe_adma_eot_handler,
3900                           0, dev_driver_string(adev->dev), chan);
3901         if (ret) {
3902                 dev_err(adev->dev, "can't request irq %d\n",
3903                         adev->irq);
3904                 *initcode = PPC_ADMA_INIT_IRQ1;
3905                 ret = -EIO;
3906                 goto err_req1;
3907         }
3908
3909         /* only DMA engines have a separate error IRQ
3910          * so it's Ok if err_irq < 0 in XOR engine case.
3911          */
3912         if (adev->err_irq > 0) {
3913                 /* both DMA engines share common error IRQ */
3914                 ret = request_irq(adev->err_irq,
3915                                   ppc440spe_adma_err_handler,
3916                                   IRQF_SHARED,
3917                                   dev_driver_string(adev->dev),
3918                                   chan);
3919                 if (ret) {
3920                         dev_err(adev->dev, "can't request irq %d\n",
3921                                 adev->err_irq);
3922                         *initcode = PPC_ADMA_INIT_IRQ2;
3923                         ret = -EIO;
3924                         goto err_req2;
3925                 }
3926         }
3927
3928         if (adev->id == PPC440SPE_XOR_ID) {
3929                 /* enable XOR engine interrupts */
3930                 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3931                             XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT,
3932                             &adev->xor_reg->ier);
3933         } else {
3934                 u32 mask, enable;
3935
3936                 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
3937                 if (!np) {
3938                         pr_err("%s: can't find I2O device tree node\n",
3939                                 __func__);
3940                         ret = -ENODEV;
3941                         goto err_req2;
3942                 }
3943                 adev->i2o_reg = of_iomap(np, 0);
3944                 if (!adev->i2o_reg) {
3945                         pr_err("%s: failed to map I2O registers\n", __func__);
3946                         of_node_put(np);
3947                         ret = -EINVAL;
3948                         goto err_req2;
3949                 }
3950                 of_node_put(np);
3951                 /* Unmask 'CS FIFO Attention' interrupts and
3952                  * enable generating interrupts on errors
3953                  */
3954                 enable = (adev->id == PPC440SPE_DMA0_ID) ?
3955                          ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3956                          ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3957                 mask = ioread32(&adev->i2o_reg->iopim) & enable;
3958                 iowrite32(mask, &adev->i2o_reg->iopim);
3959         }
3960         return 0;
3961
3962 err_req2:
3963         free_irq(adev->irq, chan);
3964 err_req1:
3965         irq_dispose_mapping(adev->irq);
3966 err_irq_map:
3967         if (adev->err_irq > 0) {
3968                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref))
3969                         irq_dispose_mapping(adev->err_irq);
3970         }
3971         return ret;
3972 }
3973
3974 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev,
3975                                         struct ppc440spe_adma_chan *chan)
3976 {
3977         u32 mask, disable;
3978
3979         if (adev->id == PPC440SPE_XOR_ID) {
3980                 /* disable XOR engine interrupts */
3981                 mask = ioread32be(&adev->xor_reg->ier);
3982                 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT |
3983                           XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT);
3984                 iowrite32be(mask, &adev->xor_reg->ier);
3985         } else {
3986                 /* disable DMAx engine interrupts */
3987                 disable = (adev->id == PPC440SPE_DMA0_ID) ?
3988                           (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) :
3989                           (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM);
3990                 mask = ioread32(&adev->i2o_reg->iopim) | disable;
3991                 iowrite32(mask, &adev->i2o_reg->iopim);
3992         }
3993         free_irq(adev->irq, chan);
3994         irq_dispose_mapping(adev->irq);
3995         if (adev->err_irq > 0) {
3996                 free_irq(adev->err_irq, chan);
3997                 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) {
3998                         irq_dispose_mapping(adev->err_irq);
3999                         iounmap(adev->i2o_reg);
4000                 }
4001         }
4002 }
4003
4004 /**
4005  * ppc440spe_adma_probe - probe the asynch device
4006  */
4007 static int ppc440spe_adma_probe(struct platform_device *ofdev)
4008 {
4009         struct device_node *np = ofdev->dev.of_node;
4010         struct resource res;
4011         struct ppc440spe_adma_device *adev;
4012         struct ppc440spe_adma_chan *chan;
4013         struct ppc_dma_chan_ref *ref, *_ref;
4014         int ret = 0, initcode = PPC_ADMA_INIT_OK;
4015         const u32 *idx;
4016         int len;
4017         void *regs;
4018         u32 id, pool_size;
4019
4020         if (of_device_is_compatible(np, "amcc,xor-accelerator")) {
4021                 id = PPC440SPE_XOR_ID;
4022                 /* As far as the XOR engine is concerned, it does not
4023                  * use FIFOs but uses linked list. So there is no dependency
4024                  * between pool size to allocate and the engine configuration.
4025                  */
4026                 pool_size = PAGE_SIZE << 1;
4027         } else {
4028                 /* it is DMA0 or DMA1 */
4029                 idx = of_get_property(np, "cell-index", &len);
4030                 if (!idx || (len != sizeof(u32))) {
4031                         dev_err(&ofdev->dev, "Device node %pOF has missing "
4032                                 "or invalid cell-index property\n",
4033                                 np);
4034                         return -EINVAL;
4035                 }
4036                 id = *idx;
4037                 /* DMA0,1 engines use FIFO to maintain CDBs, so we
4038                  * should allocate the pool accordingly to size of this
4039                  * FIFO. Thus, the pool size depends on the FIFO depth:
4040                  * how much CDBs pointers the FIFO may contain then so
4041                  * much CDBs we should provide in the pool.
4042                  * That is
4043                  *   CDB size = 32B;
4044                  *   CDBs number = (DMA0_FIFO_SIZE >> 3);
4045                  *   Pool size = CDBs number * CDB size =
4046                  *      = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2.
4047                  */
4048                 pool_size = (id == PPC440SPE_DMA0_ID) ?
4049                             DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4050                 pool_size <<= 2;
4051         }
4052
4053         if (of_address_to_resource(np, 0, &res)) {
4054                 dev_err(&ofdev->dev, "failed to get memory resource\n");
4055                 initcode = PPC_ADMA_INIT_MEMRES;
4056                 ret = -ENODEV;
4057                 goto out;
4058         }
4059
4060         if (!request_mem_region(res.start, resource_size(&res),
4061                                 dev_driver_string(&ofdev->dev))) {
4062                 dev_err(&ofdev->dev, "failed to request memory region %pR\n",
4063                         &res);
4064                 initcode = PPC_ADMA_INIT_MEMREG;
4065                 ret = -EBUSY;
4066                 goto out;
4067         }
4068
4069         /* create a device */
4070         adev = kzalloc(sizeof(*adev), GFP_KERNEL);
4071         if (!adev) {
4072                 initcode = PPC_ADMA_INIT_ALLOC;
4073                 ret = -ENOMEM;
4074                 goto err_adev_alloc;
4075         }
4076
4077         adev->id = id;
4078         adev->pool_size = pool_size;
4079         /* allocate coherent memory for hardware descriptors */
4080         adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev,
4081                                         adev->pool_size, &adev->dma_desc_pool,
4082                                         GFP_KERNEL);
4083         if (adev->dma_desc_pool_virt == NULL) {
4084                 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent "
4085                         "memory for hardware descriptors\n",
4086                         adev->pool_size);
4087                 initcode = PPC_ADMA_INIT_COHERENT;
4088                 ret = -ENOMEM;
4089                 goto err_dma_alloc;
4090         }
4091         dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n",
4092                 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool);
4093
4094         regs = ioremap(res.start, resource_size(&res));
4095         if (!regs) {
4096                 dev_err(&ofdev->dev, "failed to ioremap regs!\n");
4097                 ret = -ENOMEM;
4098                 goto err_regs_alloc;
4099         }
4100
4101         if (adev->id == PPC440SPE_XOR_ID) {
4102                 adev->xor_reg = regs;
4103                 /* Reset XOR */
4104                 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr);
4105                 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr);
4106         } else {
4107                 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ?
4108                                    DMA0_FIFO_SIZE : DMA1_FIFO_SIZE;
4109                 adev->dma_reg = regs;
4110                 /* DMAx_FIFO_SIZE is defined in bytes,
4111                  * <fsiz> - is defined in number of CDB pointers (8byte).
4112                  * DMA FIFO Length = CSlength + CPlength, where
4113                  * CSlength = CPlength = (fsiz + 1) * 8.
4114                  */
4115                 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2),
4116                           &adev->dma_reg->fsiz);
4117                 /* Configure DMA engine */
4118                 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN,
4119                           &adev->dma_reg->cfg);
4120                 /* Clear Status */
4121                 iowrite32(~0, &adev->dma_reg->dsts);
4122         }
4123
4124         adev->dev = &ofdev->dev;
4125         adev->common.dev = &ofdev->dev;
4126         INIT_LIST_HEAD(&adev->common.channels);
4127         platform_set_drvdata(ofdev, adev);
4128
4129         /* create a channel */
4130         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
4131         if (!chan) {
4132                 initcode = PPC_ADMA_INIT_CHANNEL;
4133                 ret = -ENOMEM;
4134                 goto err_chan_alloc;
4135         }
4136
4137         spin_lock_init(&chan->lock);
4138         INIT_LIST_HEAD(&chan->chain);
4139         INIT_LIST_HEAD(&chan->all_slots);
4140         chan->device = adev;
4141         chan->common.device = &adev->common;
4142         dma_cookie_init(&chan->common);
4143         list_add_tail(&chan->common.device_node, &adev->common.channels);
4144         tasklet_init(&chan->irq_tasklet, ppc440spe_adma_tasklet,
4145                      (unsigned long)chan);
4146
4147         /* allocate and map helper pages for async validation or
4148          * async_mult/async_sum_product operations on DMA0/1.
4149          */
4150         if (adev->id != PPC440SPE_XOR_ID) {
4151                 chan->pdest_page = alloc_page(GFP_KERNEL);
4152                 chan->qdest_page = alloc_page(GFP_KERNEL);
4153                 if (!chan->pdest_page ||
4154                     !chan->qdest_page) {
4155                         if (chan->pdest_page)
4156                                 __free_page(chan->pdest_page);
4157                         if (chan->qdest_page)
4158                                 __free_page(chan->qdest_page);
4159                         ret = -ENOMEM;
4160                         goto err_page_alloc;
4161                 }
4162                 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0,
4163                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4164                 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0,
4165                                            PAGE_SIZE, DMA_BIDIRECTIONAL);
4166         }
4167
4168         ref = kmalloc(sizeof(*ref), GFP_KERNEL);
4169         if (ref) {
4170                 ref->chan = &chan->common;
4171                 INIT_LIST_HEAD(&ref->node);
4172                 list_add_tail(&ref->node, &ppc440spe_adma_chan_list);
4173         } else {
4174                 dev_err(&ofdev->dev, "failed to allocate channel reference!\n");
4175                 ret = -ENOMEM;
4176                 goto err_ref_alloc;
4177         }
4178
4179         ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode);
4180         if (ret)
4181                 goto err_irq;
4182
4183         ppc440spe_adma_init_capabilities(adev);
4184
4185         ret = dma_async_device_register(&adev->common);
4186         if (ret) {
4187                 initcode = PPC_ADMA_INIT_REGISTER;
4188                 dev_err(&ofdev->dev, "failed to register dma device\n");
4189                 goto err_dev_reg;
4190         }
4191
4192         goto out;
4193
4194 err_dev_reg:
4195         ppc440spe_adma_release_irqs(adev, chan);
4196 err_irq:
4197         list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) {
4198                 if (chan == to_ppc440spe_adma_chan(ref->chan)) {
4199                         list_del(&ref->node);
4200                         kfree(ref);
4201                 }
4202         }
4203 err_ref_alloc:
4204         if (adev->id != PPC440SPE_XOR_ID) {
4205                 dma_unmap_page(&ofdev->dev, chan->pdest,
4206                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4207                 dma_unmap_page(&ofdev->dev, chan->qdest,
4208                                PAGE_SIZE, DMA_BIDIRECTIONAL);
4209                 __free_page(chan->pdest_page);
4210                 __free_page(chan->qdest_page);
4211         }
4212 err_page_alloc:
4213         kfree(chan);
4214 err_chan_alloc:
4215         if (adev->id == PPC440SPE_XOR_ID)
4216                 iounmap(adev->xor_reg);
4217         else
4218                 iounmap(adev->dma_reg);
4219 err_regs_alloc:
4220         dma_free_coherent(adev->dev, adev->pool_size,
4221                           adev->dma_desc_pool_virt,
4222                           adev->dma_desc_pool);
4223 err_dma_alloc:
4224         kfree(adev);
4225 err_adev_alloc:
4226         release_mem_region(res.start, resource_size(&res));
4227 out:
4228         if (id < PPC440SPE_ADMA_ENGINES_NUM)
4229                 ppc440spe_adma_devices[id] = initcode;
4230
4231         return ret;
4232 }
4233
4234 /**
4235  * ppc440spe_adma_remove - remove the asynch device
4236  */
4237 static int ppc440spe_adma_remove(struct platform_device *ofdev)
4238 {
4239         struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev);
4240         struct device_node *np = ofdev->dev.of_node;
4241         struct resource res;
4242         struct dma_chan *chan, *_chan;
4243         struct ppc_dma_chan_ref *ref, *_ref;
4244         struct ppc440spe_adma_chan *ppc440spe_chan;
4245
4246         if (adev->id < PPC440SPE_ADMA_ENGINES_NUM)
4247                 ppc440spe_adma_devices[adev->id] = -1;
4248
4249         dma_async_device_unregister(&adev->common);
4250
4251         list_for_each_entry_safe(chan, _chan, &adev->common.channels,
4252                                  device_node) {
4253                 ppc440spe_chan = to_ppc440spe_adma_chan(chan);
4254                 ppc440spe_adma_release_irqs(adev, ppc440spe_chan);
4255                 tasklet_kill(&ppc440spe_chan->irq_tasklet);
4256                 if (adev->id != PPC440SPE_XOR_ID) {
4257                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest,
4258                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4259                         dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest,
4260                                         PAGE_SIZE, DMA_BIDIRECTIONAL);
4261                         __free_page(ppc440spe_chan->pdest_page);
4262                         __free_page(ppc440spe_chan->qdest_page);
4263                 }
4264                 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list,
4265                                          node) {
4266                         if (ppc440spe_chan ==
4267                             to_ppc440spe_adma_chan(ref->chan)) {
4268                                 list_del(&ref->node);
4269                                 kfree(ref);
4270                         }
4271                 }
4272                 list_del(&chan->device_node);
4273                 kfree(ppc440spe_chan);
4274         }
4275
4276         dma_free_coherent(adev->dev, adev->pool_size,
4277                           adev->dma_desc_pool_virt, adev->dma_desc_pool);
4278         if (adev->id == PPC440SPE_XOR_ID)
4279                 iounmap(adev->xor_reg);
4280         else
4281                 iounmap(adev->dma_reg);
4282         of_address_to_resource(np, 0, &res);
4283         release_mem_region(res.start, resource_size(&res));
4284         kfree(adev);
4285         return 0;
4286 }
4287
4288 /*
4289  * /sys driver interface to enable h/w RAID-6 capabilities
4290  * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/
4291  * directory are "devices", "enable" and "poly".
4292  * "devices" shows available engines.
4293  * "enable" is used to enable RAID-6 capabilities or to check
4294  * whether these has been activated.
4295  * "poly" allows setting/checking used polynomial (for PPC440SPe only).
4296  */
4297
4298 static ssize_t devices_show(struct device_driver *dev, char *buf)
4299 {
4300         ssize_t size = 0;
4301         int i;
4302
4303         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) {
4304                 if (ppc440spe_adma_devices[i] == -1)
4305                         continue;
4306                 size += snprintf(buf + size, PAGE_SIZE - size,
4307                                  "PPC440SP(E)-ADMA.%d: %s\n", i,
4308                                  ppc_adma_errors[ppc440spe_adma_devices[i]]);
4309         }
4310         return size;
4311 }
4312 static DRIVER_ATTR_RO(devices);
4313
4314 static ssize_t enable_show(struct device_driver *dev, char *buf)
4315 {
4316         return snprintf(buf, PAGE_SIZE,
4317                         "PPC440SP(e) RAID-6 capabilities are %sABLED.\n",
4318                         ppc440spe_r6_enabled ? "EN" : "DIS");
4319 }
4320
4321 static ssize_t enable_store(struct device_driver *dev, const char *buf,
4322                             size_t count)
4323 {
4324         unsigned long val;
4325
4326         if (!count || count > 11)
4327                 return -EINVAL;
4328
4329         if (!ppc440spe_r6_tchan)
4330                 return -EFAULT;
4331
4332         /* Write a key */
4333         sscanf(buf, "%lx", &val);
4334         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val);
4335         isync();
4336
4337         /* Verify whether it really works now */
4338         if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) {
4339                 pr_info("PPC440SP(e) RAID-6 has been activated "
4340                         "successfully\n");
4341                 ppc440spe_r6_enabled = 1;
4342         } else {
4343                 pr_info("PPC440SP(e) RAID-6 hasn't been activated!"
4344                         " Error key ?\n");
4345                 ppc440spe_r6_enabled = 0;
4346         }
4347         return count;
4348 }
4349 static DRIVER_ATTR_RW(enable);
4350
4351 static ssize_t poly_show(struct device_driver *dev, char *buf)
4352 {
4353         ssize_t size = 0;
4354         u32 reg;
4355
4356 #ifdef CONFIG_440SP
4357         /* 440SP has fixed polynomial */
4358         reg = 0x4d;
4359 #else
4360         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4361         reg >>= MQ0_CFBHL_POLY;
4362         reg &= 0xFF;
4363 #endif
4364
4365         size = snprintf(buf, PAGE_SIZE, "PPC440SP(e) RAID-6 driver "
4366                         "uses 0x1%02x polynomial.\n", reg);
4367         return size;
4368 }
4369
4370 static ssize_t poly_store(struct device_driver *dev, const char *buf,
4371                           size_t count)
4372 {
4373         unsigned long reg, val;
4374
4375 #ifdef CONFIG_440SP
4376         /* 440SP uses default 0x14D polynomial only */
4377         return -EINVAL;
4378 #endif
4379
4380         if (!count || count > 6)
4381                 return -EINVAL;
4382
4383         /* e.g., 0x14D or 0x11D */
4384         sscanf(buf, "%lx", &val);
4385
4386         if (val & ~0x1FF)
4387                 return -EINVAL;
4388
4389         val &= 0xFF;
4390         reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL);
4391         reg &= ~(0xFF << MQ0_CFBHL_POLY);
4392         reg |= val << MQ0_CFBHL_POLY;
4393         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg);
4394
4395         return count;
4396 }
4397 static DRIVER_ATTR_RW(poly);
4398
4399 /*
4400  * Common initialisation for RAID engines; allocate memory for
4401  * DMAx FIFOs, perform configuration common for all DMA engines.
4402  * Further DMA engine specific configuration is done at probe time.
4403  */
4404 static int ppc440spe_configure_raid_devices(void)
4405 {
4406         struct device_node *np;
4407         struct resource i2o_res;
4408         struct i2o_regs __iomem *i2o_reg;
4409         dcr_host_t i2o_dcr_host;
4410         unsigned int dcr_base, dcr_len;
4411         int i, ret;
4412
4413         np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe");
4414         if (!np) {
4415                 pr_err("%s: can't find I2O device tree node\n",
4416                         __func__);
4417                 return -ENODEV;
4418         }
4419
4420         if (of_address_to_resource(np, 0, &i2o_res)) {
4421                 of_node_put(np);
4422                 return -EINVAL;
4423         }
4424
4425         i2o_reg = of_iomap(np, 0);
4426         if (!i2o_reg) {
4427                 pr_err("%s: failed to map I2O registers\n", __func__);
4428                 of_node_put(np);
4429                 return -EINVAL;
4430         }
4431
4432         /* Get I2O DCRs base */
4433         dcr_base = dcr_resource_start(np, 0);
4434         dcr_len = dcr_resource_len(np, 0);
4435         if (!dcr_base && !dcr_len) {
4436                 pr_err("%pOF: can't get DCR registers base/len!\n", np);
4437                 of_node_put(np);
4438                 iounmap(i2o_reg);
4439                 return -ENODEV;
4440         }
4441
4442         i2o_dcr_host = dcr_map(np, dcr_base, dcr_len);
4443         if (!DCR_MAP_OK(i2o_dcr_host)) {
4444                 pr_err("%pOF: failed to map DCRs!\n", np);
4445                 of_node_put(np);
4446                 iounmap(i2o_reg);
4447                 return -ENODEV;
4448         }
4449         of_node_put(np);
4450
4451         /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share
4452          * the base address of FIFO memory space.
4453          * Actually we need twice more physical memory than programmed in the
4454          * <fsiz> register (because there are two FIFOs for each DMA: CP and CS)
4455          */
4456         ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1,
4457                                          GFP_KERNEL);
4458         if (!ppc440spe_dma_fifo_buf) {
4459                 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__);
4460                 iounmap(i2o_reg);
4461                 dcr_unmap(i2o_dcr_host, dcr_len);
4462                 return -ENOMEM;
4463         }
4464
4465         /*
4466          * Configure h/w
4467          */
4468         /* Reset I2O/DMA */
4469         mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA);
4470         mtdcri(SDR0, DCRN_SDR0_SRST, 0);
4471
4472         /* Setup the base address of mmaped registers */
4473         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32));
4474         dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) |
4475                                                 I2O_REG_ENABLE);
4476         dcr_unmap(i2o_dcr_host, dcr_len);
4477
4478         /* Setup FIFO memory space base address */
4479         iowrite32(0, &i2o_reg->ifbah);
4480         iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal);
4481
4482         /* set zero FIFO size for I2O, so the whole
4483          * ppc440spe_dma_fifo_buf is used by DMAs.
4484          * DMAx_FIFOs will be configured while probe.
4485          */
4486         iowrite32(0, &i2o_reg->ifsiz);
4487         iounmap(i2o_reg);
4488
4489         /* To prepare WXOR/RXOR functionality we need access to
4490          * Memory Queue Module DCRs (finally it will be enabled
4491          * via /sys interface of the ppc440spe ADMA driver).
4492          */
4493         np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe");
4494         if (!np) {
4495                 pr_err("%s: can't find MQ device tree node\n",
4496                         __func__);
4497                 ret = -ENODEV;
4498                 goto out_free;
4499         }
4500
4501         /* Get MQ DCRs base */
4502         dcr_base = dcr_resource_start(np, 0);
4503         dcr_len = dcr_resource_len(np, 0);
4504         if (!dcr_base && !dcr_len) {
4505                 pr_err("%pOF: can't get DCR registers base/len!\n", np);
4506                 ret = -ENODEV;
4507                 goto out_mq;
4508         }
4509
4510         ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len);
4511         if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) {
4512                 pr_err("%pOF: failed to map DCRs!\n", np);
4513                 ret = -ENODEV;
4514                 goto out_mq;
4515         }
4516         of_node_put(np);
4517         ppc440spe_mq_dcr_len = dcr_len;
4518
4519         /* Set HB alias */
4520         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB);
4521
4522         /* Set:
4523          * - LL transaction passing limit to 1;
4524          * - Memory controller cycle limit to 1;
4525          * - Galois Polynomial to 0x14d (default)
4526          */
4527         dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL,
4528                   (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) |
4529                   (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY));
4530
4531         atomic_set(&ppc440spe_adma_err_irq_ref, 0);
4532         for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++)
4533                 ppc440spe_adma_devices[i] = -1;
4534
4535         return 0;
4536
4537 out_mq:
4538         of_node_put(np);
4539 out_free:
4540         kfree(ppc440spe_dma_fifo_buf);
4541         return ret;
4542 }
4543
4544 static const struct of_device_id ppc440spe_adma_of_match[] = {
4545         { .compatible   = "ibm,dma-440spe", },
4546         { .compatible   = "amcc,xor-accelerator", },
4547         {},
4548 };
4549 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match);
4550
4551 static struct platform_driver ppc440spe_adma_driver = {
4552         .probe = ppc440spe_adma_probe,
4553         .remove = ppc440spe_adma_remove,
4554         .driver = {
4555                 .name = "PPC440SP(E)-ADMA",
4556                 .of_match_table = ppc440spe_adma_of_match,
4557         },
4558 };
4559
4560 static __init int ppc440spe_adma_init(void)
4561 {
4562         int ret;
4563
4564         ret = ppc440spe_configure_raid_devices();
4565         if (ret)
4566                 return ret;
4567
4568         ret = platform_driver_register(&ppc440spe_adma_driver);
4569         if (ret) {
4570                 pr_err("%s: failed to register platform driver\n",
4571                         __func__);
4572                 goto out_reg;
4573         }
4574
4575         /* Initialization status */
4576         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4577                                  &driver_attr_devices);
4578         if (ret)
4579                 goto out_dev;
4580
4581         /* RAID-6 h/w enable entry */
4582         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4583                                  &driver_attr_enable);
4584         if (ret)
4585                 goto out_en;
4586
4587         /* GF polynomial to use */
4588         ret = driver_create_file(&ppc440spe_adma_driver.driver,
4589                                  &driver_attr_poly);
4590         if (!ret)
4591                 return ret;
4592
4593         driver_remove_file(&ppc440spe_adma_driver.driver,
4594                            &driver_attr_enable);
4595 out_en:
4596         driver_remove_file(&ppc440spe_adma_driver.driver,
4597                            &driver_attr_devices);
4598 out_dev:
4599         /* User will not be able to enable h/w RAID-6 */
4600         pr_err("%s: failed to create RAID-6 driver interface\n",
4601                 __func__);
4602         platform_driver_unregister(&ppc440spe_adma_driver);
4603 out_reg:
4604         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4605         kfree(ppc440spe_dma_fifo_buf);
4606         return ret;
4607 }
4608
4609 static void __exit ppc440spe_adma_exit(void)
4610 {
4611         driver_remove_file(&ppc440spe_adma_driver.driver,
4612                            &driver_attr_poly);
4613         driver_remove_file(&ppc440spe_adma_driver.driver,
4614                            &driver_attr_enable);
4615         driver_remove_file(&ppc440spe_adma_driver.driver,
4616                            &driver_attr_devices);
4617         platform_driver_unregister(&ppc440spe_adma_driver);
4618         dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len);
4619         kfree(ppc440spe_dma_fifo_buf);
4620 }
4621
4622 arch_initcall(ppc440spe_adma_init);
4623 module_exit(ppc440spe_adma_exit);
4624
4625 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>");
4626 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver");
4627 MODULE_LICENSE("GPL");