drivers/crypto/ccp/ccp-dmaengine.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * AMD Cryptographic Coprocessor (CCP) driver
   4  *
   5  * Copyright (C) 2016,2017 Advanced Micro Devices, Inc.
   6  *
   7  * Author: Gary R Hook <gary.hook@amd.com>
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/kernel.h>
  12 #include <linux/dmaengine.h>
  13 #include <linux/spinlock.h>
  14 #include <linux/mutex.h>
  15 #include <linux/ccp.h>
  16
  17 #include "ccp-dev.h"
  18 #include "../../dma/dmaengine.h"
  19
  20 #define CCP_DMA_WIDTH(_mask)            \
  21 ({                                      \
  22         u64 mask = _mask + 1;           \
  23         (mask == 0) ? 64 : fls64(mask); \
  24 })
  25
  26 /* The CCP as a DMA provider can be configured for public or private
  27  * channels. Default is specified in the vdata for the device (PCI ID).
  28  * This module parameter will override for all channels on all devices:
  29  *   dma_chan_attr = 0x2 to force all channels public
  30  *                 = 0x1 to force all channels private
  31  *                 = 0x0 to defer to the vdata setting
  32  *                 = any other value: warning, revert to 0x0
  33  */
  34 static unsigned int dma_chan_attr = CCP_DMA_DFLT;
  35 module_param(dma_chan_attr, uint, 0444);
  36 MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
  37
  38 static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp)
  39 {
  40         switch (dma_chan_attr) {
  41         case CCP_DMA_DFLT:
  42                 return ccp->vdata->dma_chan_attr;
  43
  44         case CCP_DMA_PRIV:
  45                 return DMA_PRIVATE;
  46
  47         case CCP_DMA_PUB:
  48                 return 0;
  49
  50         default:
  51                 dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n",
  52                               dma_chan_attr);
  53                 return ccp->vdata->dma_chan_attr;
  54         }
  55 }
  56
  57 static void ccp_free_cmd_resources(struct ccp_device *ccp,
  58                                    struct list_head *list)
  59 {
  60         struct ccp_dma_cmd *cmd, *ctmp;
  61
  62         list_for_each_entry_safe(cmd, ctmp, list, entry) {
  63                 list_del(&cmd->entry);
  64                 kmem_cache_free(ccp->dma_cmd_cache, cmd);
  65         }
  66 }
  67
  68 static void ccp_free_desc_resources(struct ccp_device *ccp,
  69                                     struct list_head *list)
  70 {
  71         struct ccp_dma_desc *desc, *dtmp;
  72
  73         list_for_each_entry_safe(desc, dtmp, list, entry) {
  74                 ccp_free_cmd_resources(ccp, &desc->active);
  75                 ccp_free_cmd_resources(ccp, &desc->pending);
  76
  77                 list_del(&desc->entry);
  78                 kmem_cache_free(ccp->dma_desc_cache, desc);
  79         }
  80 }
  81
  82 static void ccp_free_chan_resources(struct dma_chan *dma_chan)
  83 {
  84         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
  85                                                  dma_chan);
  86         unsigned long flags;
  87
  88         dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan);
  89
  90         spin_lock_irqsave(&chan->lock, flags);
  91
  92         ccp_free_desc_resources(chan->ccp, &chan->complete);
  93         ccp_free_desc_resources(chan->ccp, &chan->active);
  94         ccp_free_desc_resources(chan->ccp, &chan->pending);
  95         ccp_free_desc_resources(chan->ccp, &chan->created);
  96
  97         spin_unlock_irqrestore(&chan->lock, flags);
  98 }
  99
 100 static void ccp_cleanup_desc_resources(struct ccp_device *ccp,
 101                                        struct list_head *list)
 102 {
 103         struct ccp_dma_desc *desc, *dtmp;
 104
 105         list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {
 106                 if (!async_tx_test_ack(&desc->tx_desc))
 107                         continue;
 108
 109                 dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
 110
 111                 ccp_free_cmd_resources(ccp, &desc->active);
 112                 ccp_free_cmd_resources(ccp, &desc->pending);
 113
 114                 list_del(&desc->entry);
 115                 kmem_cache_free(ccp->dma_desc_cache, desc);
 116         }
 117 }
 118
 119 static void ccp_do_cleanup(unsigned long data)
 120 {
 121         struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data;
 122         unsigned long flags;
 123
 124         dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__,
 125                 dma_chan_name(&chan->dma_chan));
 126
 127         spin_lock_irqsave(&chan->lock, flags);
 128
 129         ccp_cleanup_desc_resources(chan->ccp, &chan->complete);
 130
 131         spin_unlock_irqrestore(&chan->lock, flags);
 132 }
 133
 134 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)
 135 {
 136         struct ccp_dma_cmd *cmd;
 137         int ret;
 138
 139         cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);
 140         list_move(&cmd->entry, &desc->active);
 141
 142         dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,
 143                 desc->tx_desc.cookie, cmd);
 144
 145         ret = ccp_enqueue_cmd(&cmd->ccp_cmd);
 146         if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY))
 147                 return 0;
 148
 149         dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,
 150                 ret, desc->tx_desc.cookie, cmd);
 151
 152         return ret;
 153 }
 154
 155 static void ccp_free_active_cmd(struct ccp_dma_desc *desc)
 156 {
 157         struct ccp_dma_cmd *cmd;
 158
 159         cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,
 160                                        entry);
 161         if (!cmd)
 162                 return;
 163
 164         dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",
 165                 __func__, desc->tx_desc.cookie, cmd);
 166
 167         list_del(&cmd->entry);
 168         kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);
 169 }
 170
 171 static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan,
 172                                                 struct ccp_dma_desc *desc)
 173 {
 174         /* Move current DMA descriptor to the complete list */
 175         if (desc)
 176                 list_move(&desc->entry, &chan->complete);
 177
 178         /* Get the next DMA descriptor on the active list */
 179         desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
 180                                         entry);
 181
 182         return desc;
 183 }
 184
 185 static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan,
 186                                                    struct ccp_dma_desc *desc)
 187 {
 188         struct dma_async_tx_descriptor *tx_desc;
 189         unsigned long flags;
 190
 191         /* Loop over descriptors until one is found with commands */
 192         do {
 193                 if (desc) {
 194                         /* Remove the DMA command from the list and free it */
 195                         ccp_free_active_cmd(desc);
 196
 197                         if (!list_empty(&desc->pending)) {
 198                                 /* No errors, keep going */
 199                                 if (desc->status != DMA_ERROR)
 200                                         return desc;
 201
 202                                 /* Error, free remaining commands and move on */
 203                                 ccp_free_cmd_resources(desc->ccp,
 204                                                        &desc->pending);
 205                         }
 206
 207                         tx_desc = &desc->tx_desc;
 208                 } else {
 209                         tx_desc = NULL;
 210                 }
 211
 212                 spin_lock_irqsave(&chan->lock, flags);
 213
 214                 if (desc) {
 215                         if (desc->status != DMA_ERROR)
 216                                 desc->status = DMA_COMPLETE;
 217
 218                         dev_dbg(desc->ccp->dev,
 219                                 "%s - tx %d complete, status=%u\n", __func__,
 220                                 desc->tx_desc.cookie, desc->status);
 221
 222                         dma_cookie_complete(tx_desc);
 223                         dma_descriptor_unmap(tx_desc);
 224                 }
 225
 226                 desc = __ccp_next_dma_desc(chan, desc);
 227
 228                 spin_unlock_irqrestore(&chan->lock, flags);
 229
 230                 if (tx_desc) {
 231                         dmaengine_desc_get_callback_invoke(tx_desc, NULL);
 232
 233                         dma_run_dependencies(tx_desc);
 234                 }
 235         } while (desc);
 236
 237         return NULL;
 238 }
 239
 240 static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan)
 241 {
 242         struct ccp_dma_desc *desc;
 243
 244         if (list_empty(&chan->pending))
 245                 return NULL;
 246
 247         desc = list_empty(&chan->active)
 248                 ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry)
 249                 : NULL;
 250
 251         list_splice_tail_init(&chan->pending, &chan->active);
 252
 253         return desc;
 254 }
 255
 256 static void ccp_cmd_callback(void *data, int err)
 257 {
 258         struct ccp_dma_desc *desc = data;
 259         struct ccp_dma_chan *chan;
 260         int ret;
 261
 262         if (err == -EINPROGRESS)
 263                 return;
 264
 265         chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,
 266                             dma_chan);
 267
 268         dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n",
 269                 __func__, desc->tx_desc.cookie, err);
 270
 271         if (err)
 272                 desc->status = DMA_ERROR;
 273
 274         while (true) {
 275                 /* Check for DMA descriptor completion */
 276                 desc = ccp_handle_active_desc(chan, desc);
 277
 278                 /* Don't submit cmd if no descriptor or DMA is paused */
 279                 if (!desc || (chan->status == DMA_PAUSED))
 280                         break;
 281
 282                 ret = ccp_issue_next_cmd(desc);
 283                 if (!ret)
 284                         break;
 285
 286                 desc->status = DMA_ERROR;
 287         }
 288
 289         tasklet_schedule(&chan->cleanup_tasklet);
 290 }
 291
 292 static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc)
 293 {
 294         struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,
 295                                                  tx_desc);
 296         struct ccp_dma_chan *chan;
 297         dma_cookie_t cookie;
 298         unsigned long flags;
 299
 300         chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan);
 301
 302         spin_lock_irqsave(&chan->lock, flags);
 303
 304         cookie = dma_cookie_assign(tx_desc);
 305         list_del(&desc->entry);
 306         list_add_tail(&desc->entry, &chan->pending);
 307
 308         spin_unlock_irqrestore(&chan->lock, flags);
 309
 310         dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n",
 311                 __func__, cookie);
 312
 313         return cookie;
 314 }
 315
 316 static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan)
 317 {
 318         struct ccp_dma_cmd *cmd;
 319
 320         cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT);
 321         if (cmd)
 322                 memset(cmd, 0, sizeof(*cmd));
 323
 324         return cmd;
 325 }
 326
 327 static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan,
 328                                                unsigned long flags)
 329 {
 330         struct ccp_dma_desc *desc;
 331
 332         desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);
 333         if (!desc)
 334                 return NULL;
 335
 336         dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);
 337         desc->tx_desc.flags = flags;
 338         desc->tx_desc.tx_submit = ccp_tx_submit;
 339         desc->ccp = chan->ccp;
 340         INIT_LIST_HEAD(&desc->pending);
 341         INIT_LIST_HEAD(&desc->active);
 342         desc->status = DMA_IN_PROGRESS;
 343
 344         return desc;
 345 }
 346
 347 static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan,
 348                                             struct scatterlist *dst_sg,
 349                                             unsigned int dst_nents,
 350                                             struct scatterlist *src_sg,
 351                                             unsigned int src_nents,
 352                                             unsigned long flags)
 353 {
 354         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 355                                                  dma_chan);
 356         struct ccp_device *ccp = chan->ccp;
 357         struct ccp_dma_desc *desc;
 358         struct ccp_dma_cmd *cmd;
 359         struct ccp_cmd *ccp_cmd;
 360         struct ccp_passthru_nomap_engine *ccp_pt;
 361         unsigned int src_offset, src_len;
 362         unsigned int dst_offset, dst_len;
 363         unsigned int len;
 364         unsigned long sflags;
 365         size_t total_len;
 366
 367         if (!dst_sg || !src_sg)
 368                 return NULL;
 369
 370         if (!dst_nents || !src_nents)
 371                 return NULL;
 372
 373         desc = ccp_alloc_dma_desc(chan, flags);
 374         if (!desc)
 375                 return NULL;
 376
 377         total_len = 0;
 378
 379         src_len = sg_dma_len(src_sg);
 380         src_offset = 0;
 381
 382         dst_len = sg_dma_len(dst_sg);
 383         dst_offset = 0;
 384
 385         while (true) {
 386                 if (!src_len) {
 387                         src_nents--;
 388                         if (!src_nents)
 389                                 break;
 390
 391                         src_sg = sg_next(src_sg);
 392                         if (!src_sg)
 393                                 break;
 394
 395                         src_len = sg_dma_len(src_sg);
 396                         src_offset = 0;
 397                         continue;
 398                 }
 399
 400                 if (!dst_len) {
 401                         dst_nents--;
 402                         if (!dst_nents)
 403                                 break;
 404
 405                         dst_sg = sg_next(dst_sg);
 406                         if (!dst_sg)
 407                                 break;
 408
 409                         dst_len = sg_dma_len(dst_sg);
 410                         dst_offset = 0;
 411                         continue;
 412                 }
 413
 414                 len = min(dst_len, src_len);
 415
 416                 cmd = ccp_alloc_dma_cmd(chan);
 417                 if (!cmd)
 418                         goto err;
 419
 420                 ccp_cmd = &cmd->ccp_cmd;
 421                 ccp_cmd->ccp = chan->ccp;
 422                 ccp_pt = &ccp_cmd->u.passthru_nomap;
 423                 ccp_cmd->flags = CCP_CMD_MAY_BACKLOG;
 424                 ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP;
 425                 ccp_cmd->engine = CCP_ENGINE_PASSTHRU;
 426                 ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
 427                 ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP;
 428                 ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset;
 429                 ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset;
 430                 ccp_pt->src_len = len;
 431                 ccp_pt->final = 1;
 432                 ccp_cmd->callback = ccp_cmd_callback;
 433                 ccp_cmd->data = desc;
 434
 435                 list_add_tail(&cmd->entry, &desc->pending);
 436
 437                 dev_dbg(ccp->dev,
 438                         "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__,
 439                         cmd, &ccp_pt->src_dma,
 440                         &ccp_pt->dst_dma, ccp_pt->src_len);
 441
 442                 total_len += len;
 443
 444                 src_len -= len;
 445                 src_offset += len;
 446
 447                 dst_len -= len;
 448                 dst_offset += len;
 449         }
 450
 451         desc->len = total_len;
 452
 453         if (list_empty(&desc->pending))
 454                 goto err;
 455
 456         dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);
 457
 458         spin_lock_irqsave(&chan->lock, sflags);
 459
 460         list_add_tail(&desc->entry, &chan->created);
 461
 462         spin_unlock_irqrestore(&chan->lock, sflags);
 463
 464         return desc;
 465
 466 err:
 467         ccp_free_cmd_resources(ccp, &desc->pending);
 468         kmem_cache_free(ccp->dma_desc_cache, desc);
 469
 470         return NULL;
 471 }
 472
 473 static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy(
 474         struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len,
 475         unsigned long flags)
 476 {
 477         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 478                                                  dma_chan);
 479         struct ccp_dma_desc *desc;
 480         struct scatterlist dst_sg, src_sg;
 481
 482         dev_dbg(chan->ccp->dev,
 483                 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
 484                 __func__, &src, &dst, len, flags);
 485
 486         sg_init_table(&dst_sg, 1);
 487         sg_dma_address(&dst_sg) = dst;
 488         sg_dma_len(&dst_sg) = len;
 489
 490         sg_init_table(&src_sg, 1);
 491         sg_dma_address(&src_sg) = src;
 492         sg_dma_len(&src_sg) = len;
 493
 494         desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);
 495         if (!desc)
 496                 return NULL;
 497
 498         return &desc->tx_desc;
 499 }
 500
 501 static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt(
 502         struct dma_chan *dma_chan, unsigned long flags)
 503 {
 504         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 505                                                  dma_chan);
 506         struct ccp_dma_desc *desc;
 507
 508         desc = ccp_alloc_dma_desc(chan, flags);
 509         if (!desc)
 510                 return NULL;
 511
 512         return &desc->tx_desc;
 513 }
 514
 515 static void ccp_issue_pending(struct dma_chan *dma_chan)
 516 {
 517         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 518                                                  dma_chan);
 519         struct ccp_dma_desc *desc;
 520         unsigned long flags;
 521
 522         dev_dbg(chan->ccp->dev, "%s\n", __func__);
 523
 524         spin_lock_irqsave(&chan->lock, flags);
 525
 526         desc = __ccp_pending_to_active(chan);
 527
 528         spin_unlock_irqrestore(&chan->lock, flags);
 529
 530         /* If there was nothing active, start processing */
 531         if (desc)
 532                 ccp_cmd_callback(desc, 0);
 533 }
 534
 535 static enum dma_status ccp_tx_status(struct dma_chan *dma_chan,
 536                                      dma_cookie_t cookie,
 537                                      struct dma_tx_state *state)
 538 {
 539         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 540                                                  dma_chan);
 541         struct ccp_dma_desc *desc;
 542         enum dma_status ret;
 543         unsigned long flags;
 544
 545         if (chan->status == DMA_PAUSED) {
 546                 ret = DMA_PAUSED;
 547                 goto out;
 548         }
 549
 550         ret = dma_cookie_status(dma_chan, cookie, state);
 551         if (ret == DMA_COMPLETE) {
 552                 spin_lock_irqsave(&chan->lock, flags);
 553
 554                 /* Get status from complete chain, if still there */
 555                 list_for_each_entry(desc, &chan->complete, entry) {
 556                         if (desc->tx_desc.cookie != cookie)
 557                                 continue;
 558
 559                         ret = desc->status;
 560                         break;
 561                 }
 562
 563                 spin_unlock_irqrestore(&chan->lock, flags);
 564         }
 565
 566 out:
 567         dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret);
 568
 569         return ret;
 570 }
 571
 572 static int ccp_pause(struct dma_chan *dma_chan)
 573 {
 574         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 575                                                  dma_chan);
 576
 577         chan->status = DMA_PAUSED;
 578
 579         /*TODO: Wait for active DMA to complete before returning? */
 580
 581         return 0;
 582 }
 583
 584 static int ccp_resume(struct dma_chan *dma_chan)
 585 {
 586         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 587                                                  dma_chan);
 588         struct ccp_dma_desc *desc;
 589         unsigned long flags;
 590
 591         spin_lock_irqsave(&chan->lock, flags);
 592
 593         desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,
 594                                         entry);
 595
 596         spin_unlock_irqrestore(&chan->lock, flags);
 597
 598         /* Indicate the channel is running again */
 599         chan->status = DMA_IN_PROGRESS;
 600
 601         /* If there was something active, re-start */
 602         if (desc)
 603                 ccp_cmd_callback(desc, 0);
 604
 605         return 0;
 606 }
 607
 608 static int ccp_terminate_all(struct dma_chan *dma_chan)
 609 {
 610         struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan,
 611                                                  dma_chan);
 612         unsigned long flags;
 613
 614         dev_dbg(chan->ccp->dev, "%s\n", __func__);
 615
 616         /*TODO: Wait for active DMA to complete before continuing */
 617
 618         spin_lock_irqsave(&chan->lock, flags);
 619
 620         /*TODO: Purge the complete list? */
 621         ccp_free_desc_resources(chan->ccp, &chan->active);
 622         ccp_free_desc_resources(chan->ccp, &chan->pending);
 623         ccp_free_desc_resources(chan->ccp, &chan->created);
 624
 625         spin_unlock_irqrestore(&chan->lock, flags);
 626
 627         return 0;
 628 }
 629
 630 int ccp_dmaengine_register(struct ccp_device *ccp)
 631 {
 632         struct ccp_dma_chan *chan;
 633         struct dma_device *dma_dev = &ccp->dma_dev;
 634         struct dma_chan *dma_chan;
 635         char *dma_cmd_cache_name;
 636         char *dma_desc_cache_name;
 637         unsigned int i;
 638         int ret;
 639
 640         ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count,
 641                                          sizeof(*(ccp->ccp_dma_chan)),
 642                                          GFP_KERNEL);
 643         if (!ccp->ccp_dma_chan)
 644                 return -ENOMEM;
 645
 646         dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 647                                             "%s-dmaengine-cmd-cache",
 648                                             ccp->name);
 649         if (!dma_cmd_cache_name)
 650                 return -ENOMEM;
 651
 652         ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name,
 653                                                sizeof(struct ccp_dma_cmd),
 654                                                sizeof(void *),
 655                                                SLAB_HWCACHE_ALIGN, NULL);
 656         if (!ccp->dma_cmd_cache)
 657                 return -ENOMEM;
 658
 659         dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL,
 660                                              "%s-dmaengine-desc-cache",
 661                                              ccp->name);
 662         if (!dma_desc_cache_name) {
 663                 ret = -ENOMEM;
 664                 goto err_cache;
 665         }
 666
 667         ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name,
 668                                                 sizeof(struct ccp_dma_desc),
 669                                                 sizeof(void *),
 670                                                 SLAB_HWCACHE_ALIGN, NULL);
 671         if (!ccp->dma_desc_cache) {
 672                 ret = -ENOMEM;
 673                 goto err_cache;
 674         }
 675
 676         dma_dev->dev = ccp->dev;
 677         dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
 678         dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev));
 679         dma_dev->directions = DMA_MEM_TO_MEM;
 680         dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
 681         dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
 682         dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask);
 683
 684         /* The DMA channels for this device can be set to public or private,
 685          * and overridden by the module parameter dma_chan_attr.
 686          * Default: according to the value in vdata (dma_chan_attr=0)
 687          * dma_chan_attr=0x1: all channels private (override vdata)
 688          * dma_chan_attr=0x2: all channels public (override vdata)
 689          */
 690         if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE)
 691                 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
 692
 693         INIT_LIST_HEAD(&dma_dev->channels);
 694         for (i = 0; i < ccp->cmd_q_count; i++) {
 695                 chan = ccp->ccp_dma_chan + i;
 696                 dma_chan = &chan->dma_chan;
 697
 698                 chan->ccp = ccp;
 699
 700                 spin_lock_init(&chan->lock);
 701                 INIT_LIST_HEAD(&chan->created);
 702                 INIT_LIST_HEAD(&chan->pending);
 703                 INIT_LIST_HEAD(&chan->active);
 704                 INIT_LIST_HEAD(&chan->complete);
 705
 706                 tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup,
 707                              (unsigned long)chan);
 708
 709                 dma_chan->device = dma_dev;
 710                 dma_cookie_init(dma_chan);
 711
 712                 list_add_tail(&dma_chan->device_node, &dma_dev->channels);
 713         }
 714
 715         dma_dev->device_free_chan_resources = ccp_free_chan_resources;
 716         dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy;
 717         dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt;
 718         dma_dev->device_issue_pending = ccp_issue_pending;
 719         dma_dev->device_tx_status = ccp_tx_status;
 720         dma_dev->device_pause = ccp_pause;
 721         dma_dev->device_resume = ccp_resume;
 722         dma_dev->device_terminate_all = ccp_terminate_all;
 723
 724         ret = dma_async_device_register(dma_dev);
 725         if (ret)
 726                 goto err_reg;
 727
 728         return 0;
 729
 730 err_reg:
 731         kmem_cache_destroy(ccp->dma_desc_cache);
 732
 733 err_cache:
 734         kmem_cache_destroy(ccp->dma_cmd_cache);
 735
 736         return ret;
 737 }
 738
 739 void ccp_dmaengine_unregister(struct ccp_device *ccp)
 740 {
 741         struct dma_device *dma_dev = &ccp->dma_dev;
 742
 743         dma_async_device_unregister(dma_dev);
 744
 745         kmem_cache_destroy(ccp->dma_desc_cache);
 746         kmem_cache_destroy(ccp->dma_cmd_cache);
 747 }