drivers/crypto/ccp/ccp-dmaengine.c

   1 /*
   2  * AMD Cryptographic Coprocessor (CCP) driver
   3  *
   4  * Copyright (C) 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Gary R Hook <gary.hook@amd.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/kernel.h>
  14 #include <linux/dmaengine.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/mutex.h>
  17 #include <linux/ccp.h>
  18
  19 #include "ccp-dev.h"
  20 #include "../../dma/dmaengine.h"
  21
  22 #define CCP_DMA_WIDTH(_mask)            \
  23 ({                                      \
  24         u64 mask = _mask + 1;           \
  25         (mask == 0) ? 64 : fls64(mask); \
  26 })
  27
  28 static void ccp_free_cmd_resources(struct ccp_device *ccp,
  29                                    struct list_head *list)
  30 {
  31         struct ccp_dma_cmd *cmd, *ctmp;
  32
  33         list_for_each_entry_safe(cmd, ctmp, list, entry) {
  34                 list_del(&cmd->entry);
  35                 kmem_cache_free(ccp->dma_cmd_cache, cmd);
  36         }
  37 }
  38
  39 static void ccp_free_desc_resources(struct ccp_device *ccp,
  40                                     struct list_head *list)
  41 {
  42         struct ccp_dma_desc *desc, *dtmp;
  43
  44         list_for_each_entry_safe(desc, dtmp, list, entry) {
  45                 ccp_free_cmd_resources(ccp, &desc->active);
  46                 ccp_free_cmd_resources(ccp, &desc->pending);
  47
  48                 list_del(&desc->entry);
  49                 kmem_cache_free(ccp->dma_desc_cache, desc);
  50         }
  51 }
  52
  53 static void ccp_free_chan_resources(struct dma_chan *dma_chan)
  54 {
  55         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  56                                                  dma_chan);
  57         unsigned long flags;
  58
  59         dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan);
  60
  61         spin_lock_irqsave(&chan->lock, flags);
  62
  63         ccp_free_desc_resources(chan->ccp, &chan->complete);
  64         ccp_free_desc_resources(chan->ccp, &chan->active);
  65         ccp_free_desc_resources(chan->ccp, &chan->pending);
  66
  67         spin_unlock_irqrestore(&chan->lock, flags);
  68 }
  69
  70 static void ccp_cleanup_desc_resources(struct ccp_device *ccp,
  71                                        struct list_head *list)
  72 {
  73         struct ccp_dma_desc *desc, *dtmp;
  74
  75         list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {
  76                 if (!async_tx_test_ack(&desc->tx_desc))
  77                         continue;
  78
  79                 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
  80
  81                 ccp_free_cmd_resources(ccp, &desc->active);
  82                 ccp_free_cmd_resources(ccp, &desc->pending);
  83
  84                 list_del(&desc->entry);
  85                 kmem_cache_free(ccp->dma_desc_cache, desc);
  86         }
  87 }
  88
  89 static void ccp_do_cleanup(unsigned long data)
  90 {
  91         struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data;
  92         unsigned long flags;
  93
  94         dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__,
  95                 dma_chan_name(&chan->dma_chan));
  96
  97         spin_lock_irqsave(&chan->lock, flags);
  98
  99         ccp_cleanup_desc_resources(chan->ccp, &chan->complete);
 100
 101         spin_unlock_irqrestore(&chan->lock, flags);
 102 }
 103
 104 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)
 105 {
 106         struct ccp_dma_cmd *cmd;
 107         int ret;
 108
 109         cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);
 110         list_move(&cmd->entry, &desc->active);
 111
 112         dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,
 113                 desc->tx_desc.cookie, cmd);
 114
 115         ret = ccp_enqueue_cmd(&cmd->ccp_cmd);
 116         if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY))
 117                 return 0;
 118
 119         dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,
 120                 ret, desc->tx_desc.cookie, cmd);
 121
 122         return ret;
 123 }
 124
 125 static void ccp_free_active_cmd(struct ccp_dma_desc *desc)
 126 {
 127         struct ccp_dma_cmd *cmd;
 128
 129         cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,
 130                                        entry);
 131         if (!cmd)
 132                 return;
 133
 134         dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",
 135                 __func__, desc->tx_desc.cookie, cmd);
 136
 137         list_del(&cmd->entry);
 138         kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);
 139 }
 140
 141 static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan,
 142                                                 struct ccp_dma_desc *desc)
 143 {
 144         /* Move current DMA descriptor to the complete list */
 145         if (desc)
 146                 list_move(&desc->entry, &chan->complete);
 147
 148         /* Get the next DMA descriptor on the active list */
 149         desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
 150                                         entry);
 151
 152         return desc;
 153 }
 154
 155 static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan,
 156                                                    struct ccp_dma_desc *desc)
 157 {
 158         struct dma_async_tx_descriptor *tx_desc;
 159         unsigned long flags;
 160
 161         /* Loop over descriptors until one is found with commands */
 162         do {
 163                 if (desc) {
 164                         /* Remove the DMA command from the list and free it */
 165                         ccp_free_active_cmd(desc);
 166
 167                         if (!list_empty(&desc->pending)) {
 168                                 /* No errors, keep going */
 169                                 if (desc->status != DMA_ERROR)
 170                                         return desc;
 171
 172                                 /* Error, free remaining commands and move on */
 173                                 ccp_free_cmd_resources(desc->ccp,
 174                                                        &desc->pending);
 175                         }
 176
 177                         tx_desc = &desc->tx_desc;
 178                 } else {
 179                         tx_desc = NULL;
 180                 }
 181
 182                 spin_lock_irqsave(&chan->lock, flags);
 183
 184                 if (desc) {
 185                         if (desc->status != DMA_ERROR)
 186                                 desc->status = DMA_COMPLETE;
 187
 188                         dev_dbg(desc->ccp->dev,
 189                                 "%s - tx %d complete, status=%u\n", __func__,
 190                                 desc->tx_desc.cookie, desc->status);
 191
 192                         dma_cookie_complete(tx_desc);
 193                 }
 194
 195                 desc = __ccp_next_dma_desc(chan, desc);
 196
 197                 spin_unlock_irqrestore(&chan->lock, flags);
 198
 199                 if (tx_desc) {
 200                         if (tx_desc->callback &&
 201                             (tx_desc->flags & DMA_PREP_INTERRUPT))
 202                                 tx_desc->callback(tx_desc->callback_param);
 203
 204                         dma_run_dependencies(tx_desc);
 205                 }
 206         } while (desc);
 207
 208         return NULL;
 209 }
 210
 211 static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan)
 212 {
 213         struct ccp_dma_desc *desc;
 214
 215         if (list_empty(&chan->pending))
 216                 return NULL;
 217
 218         desc = list_empty(&chan->active)
 219                 ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry)
 220                 : NULL;
 221
 222         list_splice_tail_init(&chan->pending, &chan->active);
 223
 224         return desc;
 225 }
 226
 227 static void ccp_cmd_callback(void *data, int err)
 228 {
 229         struct ccp_dma_desc *desc = data;
 230         struct ccp_dma_chan *chan;
 231         int ret;
 232
 233         if (err == -EINPROGRESS)
 234                 return;
 235
 236         chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,
 237                             dma_chan);
 238
 239         dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n",
 240                 __func__, desc->tx_desc.cookie, err);
 241
 242         if (err)
 243                 desc->status = DMA_ERROR;
 244
 245         while (true) {
 246                 /* Check for DMA descriptor completion */
 247                 desc = ccp_handle_active_desc(chan, desc);
 248
 249                 /* Don't submit cmd if no descriptor or DMA is paused */
 250                 if (!desc || (chan->status == DMA_PAUSED))
 251                         break;
 252
 253                 ret = ccp_issue_next_cmd(desc);
 254                 if (!ret)
 255                         break;
 256
 257                 desc->status = DMA_ERROR;
 258         }
 259
 260         tasklet_schedule(&chan->cleanup_tasklet);
 261 }
 262
 263 static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
 264 {
 265         struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,
 266                                                  tx_desc);
 267         struct ccp_dma_chan *chan;
 268         dma_cookie_t cookie;
 269         unsigned long flags;
 270
 271         chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan);
 272
 273         spin_lock_irqsave(&chan->lock, flags);
 274
 275         cookie = dma_cookie_assign(tx_desc);
 276         list_add_tail(&desc->entry, &chan->pending);
 277
 278         spin_unlock_irqrestore(&chan->lock, flags);
 279
 280         dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n",
 281                 __func__, cookie);
 282
 283         return cookie;
 284 }
 285
 286 static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan)
 287 {
 288         struct ccp_dma_cmd *cmd;
 289
 290         cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT);
 291         if (cmd)
 292                 memset(cmd, 0, sizeof(*cmd));
 293
 294         return cmd;
 295 }
 296
 297 static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
 298                                                unsigned long flags)
 299 {
 300         struct ccp_dma_desc *desc;
 301
 302         desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
 303         if (!desc)
 304                 return NULL;
 305
 306         dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
 307         desc->tx_desc.flags = flags;
 308         desc->tx_desc.tx_submit = ccp_tx_submit;
 309         desc->ccp = chan->ccp;
 310         INIT_LIST_HEAD(&desc->pending);
 311         INIT_LIST_HEAD(&desc->active);
 312         desc->status = DMA_IN_PROGRESS;
 313
 314         return desc;
 315 }
 316
 317 static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
 318                                             struct scatterlist *dst_sg,
 319                                             unsigned int dst_nents,
 320                                             struct scatterlist *src_sg,
 321                                             unsigned int src_nents,
 322                                             unsigned long flags)
 323 {
 324         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 325                                                  dma_chan);
 326         struct ccp_device *ccp = chan->ccp;
 327         struct ccp_dma_desc *desc;
 328         struct ccp_dma_cmd *cmd;
 329         struct ccp_cmd *ccp_cmd;
 330         struct ccp_passthru_nomap_engine *ccp_pt;
 331         unsigned int src_offset, src_len;
 332         unsigned int dst_offset, dst_len;
 333         unsigned int len;
 334         unsigned long sflags;
 335         size_t total_len;
 336
 337         if (!dst_sg || !src_sg)
 338                 return NULL;
 339
 340         if (!dst_nents || !src_nents)
 341                 return NULL;
 342
 343         desc = ccp_alloc_dma_desc(chan, flags);
 344         if (!desc)
 345                 return NULL;
 346
 347         total_len = 0;
 348
 349         src_len = sg_dma_len(src_sg);
 350         src_offset = 0;
 351
 352         dst_len = sg_dma_len(dst_sg);
 353         dst_offset = 0;
 354
 355         while (true) {
 356                 if (!src_len) {
 357                         src_nents--;
 358                         if (!src_nents)
 359                                 break;
 360
 361                         src_sg = sg_next(src_sg);
 362                         if (!src_sg)
 363                                 break;
 364
 365                         src_len = sg_dma_len(src_sg);
 366                         src_offset = 0;
 367                         continue;
 368                 }
 369
 370                 if (!dst_len) {
 371                         dst_nents--;
 372                         if (!dst_nents)
 373                                 break;
 374
 375                         dst_sg = sg_next(dst_sg);
 376                         if (!dst_sg)
 377                                 break;
 378
 379                         dst_len = sg_dma_len(dst_sg);
 380                         dst_offset = 0;
 381                         continue;
 382                 }
 383
 384                 len = min(dst_len, src_len);
 385
 386                 cmd = ccp_alloc_dma_cmd(chan);
 387                 if (!cmd)
 388                         goto err;
 389
 390                 ccp_cmd = &cmd->ccp_cmd;
 391                 ccp_pt = &ccp_cmd->u.passthru_nomap;
 392                 ccp_cmd->flags = CCP_CMD_MAY_BACKLOG;
 393                 ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP;
 394                 ccp_cmd->engine = CCP_ENGINE_PASSTHRU;
 395                 ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
 396                 ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
 397                 ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset;
 398                 ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset;
 399                 ccp_pt->src_len = len;
 400                 ccp_pt->final = 1;
 401                 ccp_cmd->callback = ccp_cmd_callback;
 402                 ccp_cmd->data = desc;
 403
 404                 list_add_tail(&cmd->entry, &desc->pending);
 405
 406                 dev_dbg(ccp->dev,
 407                         "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__,
 408                         cmd, &ccp_pt->src_dma,
 409                         &ccp_pt->dst_dma, ccp_pt->src_len);
 410
 411                 total_len += len;
 412
 413                 src_len -= len;
 414                 src_offset += len;
 415
 416                 dst_len -= len;
 417                 dst_offset += len;
 418         }
 419
 420         desc->len = total_len;
 421
 422         if (list_empty(&desc->pending))
 423                 goto err;
 424
 425         dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
 426
 427         spin_lock_irqsave(&chan->lock, sflags);
 428
 429         list_add_tail(&desc->entry, &chan->pending);
 430
 431         spin_unlock_irqrestore(&chan->lock, sflags);
 432
 433         return desc;
 434
 435 err:
 436         ccp_free_cmd_resources(ccp, &desc->pending);
 437         kmem_cache_free(ccp->dma_desc_cache, desc);
 438
 439         return NULL;
 440 }
 441
 442 static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
 443         struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len,
 444         unsigned long flags)
 445 {
 446         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 447                                                  dma_chan);
 448         struct ccp_dma_desc *desc;
 449         struct scatterlist dst_sg, src_sg;
 450
 451         dev_dbg(chan->ccp->dev,
 452                 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
 453                 __func__, &src, &dst, len, flags);
 454
 455         sg_init_table(&dst_sg, 1);
 456         sg_dma_address(&dst_sg) = dst;
 457         sg_dma_len(&dst_sg) = len;
 458
 459         sg_init_table(&src_sg, 1);
 460         sg_dma_address(&src_sg) = src;
 461         sg_dma_len(&src_sg) = len;
 462
 463         desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);
 464         if (!desc)
 465                 return NULL;
 466
 467         return &desc->tx_desc;
 468 }
 469
 470 static struct dma_async_tx_descriptor *ccp_prep_dma_sg(
 471         struct dma_chan *dma_chan, struct scatterlist *dst_sg,
 472         unsigned int dst_nents, struct scatterlist *src_sg,
 473         unsigned int src_nents, unsigned long flags)
 474 {
 475         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 476                                                  dma_chan);
 477         struct ccp_dma_desc *desc;
 478
 479         dev_dbg(chan->ccp->dev,
 480                 "%s - src=%p, src_nents=%u dst=%p, dst_nents=%u, flags=%#lx\n",
 481                 __func__, src_sg, src_nents, dst_sg, dst_nents, flags);
 482
 483         desc = ccp_create_desc(dma_chan, dst_sg, dst_nents, src_sg, src_nents,
 484                                flags);
 485         if (!desc)
 486                 return NULL;
 487
 488         return &desc->tx_desc;
 489 }
 490
 491 static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
 492         struct dma_chan *dma_chan, unsigned long flags)
 493 {
 494         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 495                                                  dma_chan);
 496         struct ccp_dma_desc *desc;
 497
 498         desc = ccp_alloc_dma_desc(chan, flags);
 499         if (!desc)
 500                 return NULL;
 501
 502         return &desc->tx_desc;
 503 }
 504
 505 static void ccp_issue_pending(struct dma_chan *dma_chan)
 506 {
 507         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 508                                                  dma_chan);
 509         struct ccp_dma_desc *desc;
 510         unsigned long flags;
 511
 512         dev_dbg(chan->ccp->dev, "%s\n", __func__);
 513
 514         spin_lock_irqsave(&chan->lock, flags);
 515
 516         desc = __ccp_pending_to_active(chan);
 517
 518         spin_unlock_irqrestore(&chan->lock, flags);
 519
 520         /* If there was nothing active, start processing */
 521         if (desc)
 522                 ccp_cmd_callback(desc, 0);
 523 }
 524
 525 static enum dma_status ccp_tx_status(struct dma_chan *dma_chan,
 526                                      dma_cookie_t cookie,
 527                                      struct dma_tx_state *state)
 528 {
 529         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 530                                                  dma_chan);
 531         struct ccp_dma_desc *desc;
 532         enum dma_status ret;
 533         unsigned long flags;
 534
 535         if (chan->status == DMA_PAUSED) {
 536                 ret = DMA_PAUSED;
 537                 goto out;
 538         }
 539
 540         ret = dma_cookie_status(dma_chan, cookie, state);
 541         if (ret == DMA_COMPLETE) {
 542                 spin_lock_irqsave(&chan->lock, flags);
 543
 544                 /* Get status from complete chain, if still there */
 545                 list_for_each_entry(desc, &chan->complete, entry) {
 546                         if (desc->tx_desc.cookie != cookie)
 547                                 continue;
 548
 549                         ret = desc->status;
 550                         break;
 551                 }
 552
 553                 spin_unlock_irqrestore(&chan->lock, flags);
 554         }
 555
 556 out:
 557         dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret);
 558
 559         return ret;
 560 }
 561
 562 static int ccp_pause(struct dma_chan *dma_chan)
 563 {
 564         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 565                                                  dma_chan);
 566
 567         chan->status = DMA_PAUSED;
 568
 569         /*TODO: Wait for active DMA to complete before returning? */
 570
 571         return 0;
 572 }
 573
 574 static int ccp_resume(struct dma_chan *dma_chan)
 575 {
 576         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 577                                                  dma_chan);
 578         struct ccp_dma_desc *desc;
 579         unsigned long flags;
 580
 581         spin_lock_irqsave(&chan->lock, flags);
 582
 583         desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
 584                                         entry);
 585
 586         spin_unlock_irqrestore(&chan->lock, flags);
 587
 588         /* Indicate the channel is running again */
 589         chan->status = DMA_IN_PROGRESS;
 590
 591         /* If there was something active, re-start */
 592         if (desc)
 593                 ccp_cmd_callback(desc, 0);
 594
 595         return 0;
 596 }
 597
 598 static int ccp_terminate_all(struct dma_chan *dma_chan)
 599 {
 600         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 601                                                  dma_chan);
 602         unsigned long flags;
 603
 604         dev_dbg(chan->ccp->dev, "%s\n", __func__);
 605
 606         /*TODO: Wait for active DMA to complete before continuing */
 607
 608         spin_lock_irqsave(&chan->lock, flags);
 609
 610         /*TODO: Purge the complete list? */
 611         ccp_free_desc_resources(chan->ccp, &chan->active);
 612         ccp_free_desc_resources(chan->ccp, &chan->pending);
 613
 614         spin_unlock_irqrestore(&chan->lock, flags);
 615
 616         return 0;
 617 }
 618
 619 int ccp_dmaengine_register(struct ccp_device *ccp)
 620 {
 621         struct ccp_dma_chan *chan;
 622         struct dma_device *dma_dev = &ccp->dma_dev;
 623         struct dma_chan *dma_chan;
 624         char *dma_cmd_cache_name;
 625         char *dma_desc_cache_name;
 626         unsigned int i;
 627         int ret;
 628
 629         ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
 630                                          sizeof(*(ccp->ccp_dma_chan)),
 631                                          GFP_KERNEL);
 632         if (!ccp->ccp_dma_chan)
 633                 return -ENOMEM;
 634
 635         dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 636                                             "%s-dmaengine-cmd-cache",
 637                                             ccp->name);
 638         if (!dma_cmd_cache_name)
 639                 return -ENOMEM;
 640
 641         ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name,
 642                                                sizeof(struct ccp_dma_cmd),
 643                                                sizeof(void *),
 644                                                SLAB_HWCACHE_ALIGN, NULL);
 645         if (!ccp->dma_cmd_cache)
 646                 return -ENOMEM;
 647
 648         dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 649                                              "%s-dmaengine-desc-cache",
 650                                              ccp->name);
 651         if (!dma_desc_cache_name) {
 652                 ret = -ENOMEM;
 653                 goto err_cache;
 654         }
 655
 656         ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
 657                                                 sizeof(struct ccp_dma_desc),
 658                                                 sizeof(void *),
 659                                                 SLAB_HWCACHE_ALIGN, NULL);
 660         if (!ccp->dma_desc_cache) {
 661                 ret = -ENOMEM;
 662                 goto err_cache;
 663         }
 664
 665         dma_dev->dev = ccp->dev;
 666         dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
 667         dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
 668         dma_dev->directions = DMA_MEM_TO_MEM;
 669         dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 670         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
 671         dma_cap_set(DMA_SG, dma_dev->cap_mask);
 672         dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
 673
 674         INIT_LIST_HEAD(&dma_dev->channels);
 675         for (i = 0; i < ccp->cmd_q_count; i++) {
 676                 chan = ccp->ccp_dma_chan + i;
 677                 dma_chan = &chan->dma_chan;
 678
 679                 chan->ccp = ccp;
 680
 681                 spin_lock_init(&chan->lock);
 682                 INIT_LIST_HEAD(&chan->pending);
 683                 INIT_LIST_HEAD(&chan->active);
 684                 INIT_LIST_HEAD(&chan->complete);
 685
 686                 tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup,
 687                              (unsigned long)chan);
 688
 689                 dma_chan->device = dma_dev;
 690                 dma_cookie_init(dma_chan);
 691
 692                 list_add_tail(&dma_chan->device_node, &dma_dev->channels);
 693         }
 694
 695         dma_dev->device_free_chan_resources = ccp_free_chan_resources;
 696         dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
 697         dma_dev->device_prep_dma_sg = ccp_prep_dma_sg;
 698         dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
 699         dma_dev->device_issue_pending = ccp_issue_pending;
 700         dma_dev->device_tx_status = ccp_tx_status;
 701         dma_dev->device_pause = ccp_pause;
 702         dma_dev->device_resume = ccp_resume;
 703         dma_dev->device_terminate_all = ccp_terminate_all;
 704
 705         ret = dma_async_device_register(dma_dev);
 706         if (ret)
 707                 goto err_reg;
 708
 709         return 0;
 710
 711 err_reg:
 712         kmem_cache_destroy(ccp->dma_desc_cache);
 713
 714 err_cache:
 715         kmem_cache_destroy(ccp->dma_cmd_cache);
 716
 717         return ret;
 718 }
 719
 720 void ccp_dmaengine_unregister(struct ccp_device *ccp)
 721 {
 722         struct dma_device *dma_dev = &ccp->dma_dev;
 723
 724         dma_async_device_unregister(dma_dev);
 725
 726         kmem_cache_destroy(ccp->dma_desc_cache);
 727         kmem_cache_destroy(ccp->dma_cmd_cache);
 728 }