drivers/dma/img-mdc-dma.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * IMG Multi-threaded DMA Controller (MDC)
   4  *
   5  * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
   6  * Copyright (C) 2014 Google, Inc.
   7  */
   8
   9 #include <linux/clk.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/dmapool.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/io.h>
  15 #include <linux/irq.h>
  16 #include <linux/kernel.h>
  17 #include <linux/mfd/syscon.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/of_device.h>
  21 #include <linux/of_dma.h>
  22 #include <linux/platform_device.h>
  23 #include <linux/pm_runtime.h>
  24 #include <linux/regmap.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27
  28 #include "dmaengine.h"
  29 #include "virt-dma.h"
  30
  31 #define MDC_MAX_DMA_CHANNELS                    32
  32
  33 #define MDC_GENERAL_CONFIG                      0x000
  34 #define MDC_GENERAL_CONFIG_LIST_IEN             BIT(31)
  35 #define MDC_GENERAL_CONFIG_IEN                  BIT(29)
  36 #define MDC_GENERAL_CONFIG_LEVEL_INT            BIT(28)
  37 #define MDC_GENERAL_CONFIG_INC_W                BIT(12)
  38 #define MDC_GENERAL_CONFIG_INC_R                BIT(8)
  39 #define MDC_GENERAL_CONFIG_PHYSICAL_W           BIT(7)
  40 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT        4
  41 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK         0x7
  42 #define MDC_GENERAL_CONFIG_PHYSICAL_R           BIT(3)
  43 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT        0
  44 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK         0x7
  45
  46 #define MDC_READ_PORT_CONFIG                    0x004
  47 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT      28
  48 #define MDC_READ_PORT_CONFIG_STHREAD_MASK       0xf
  49 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT      24
  50 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK       0xf
  51 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT      16
  52 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK       0xf
  53 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT   4
  54 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK    0xff
  55 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE        BIT(1)
  56
  57 #define MDC_READ_ADDRESS                        0x008
  58
  59 #define MDC_WRITE_ADDRESS                       0x00c
  60
  61 #define MDC_TRANSFER_SIZE                       0x010
  62 #define MDC_TRANSFER_SIZE_MASK                  0xffffff
  63
  64 #define MDC_LIST_NODE_ADDRESS                   0x014
  65
  66 #define MDC_CMDS_PROCESSED                      0x018
  67 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
  68 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK  0x3f
  69 #define MDC_CMDS_PROCESSED_INT_ACTIVE           BIT(8)
  70 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT      0
  71 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK       0x3f
  72
  73 #define MDC_CONTROL_AND_STATUS                  0x01c
  74 #define MDC_CONTROL_AND_STATUS_CANCEL           BIT(20)
  75 #define MDC_CONTROL_AND_STATUS_LIST_EN          BIT(4)
  76 #define MDC_CONTROL_AND_STATUS_EN               BIT(0)
  77
  78 #define MDC_ACTIVE_TRANSFER_SIZE                0x030
  79
  80 #define MDC_GLOBAL_CONFIG_A                             0x900
  81 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT       16
  82 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK        0xff
  83 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT          8
  84 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK           0xff
  85 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT         0
  86 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK          0xff
  87
  88 struct mdc_hw_list_desc {
  89         u32 gen_conf;
  90         u32 readport_conf;
  91         u32 read_addr;
  92         u32 write_addr;
  93         u32 xfer_size;
  94         u32 node_addr;
  95         u32 cmds_done;
  96         u32 ctrl_status;
  97         /*
  98          * Not part of the list descriptor, but instead used by the CPU to
  99          * traverse the list.
 100          */
 101         struct mdc_hw_list_desc *next_desc;
 102 };
 103
 104 struct mdc_tx_desc {
 105         struct mdc_chan *chan;
 106         struct virt_dma_desc vd;
 107         dma_addr_t list_phys;
 108         struct mdc_hw_list_desc *list;
 109         bool cyclic;
 110         bool cmd_loaded;
 111         unsigned int list_len;
 112         unsigned int list_period_len;
 113         size_t list_xfer_size;
 114         unsigned int list_cmds_done;
 115 };
 116
 117 struct mdc_chan {
 118         struct mdc_dma *mdma;
 119         struct virt_dma_chan vc;
 120         struct dma_slave_config config;
 121         struct mdc_tx_desc *desc;
 122         int irq;
 123         unsigned int periph;
 124         unsigned int thread;
 125         unsigned int chan_nr;
 126 };
 127
 128 struct mdc_dma_soc_data {
 129         void (*enable_chan)(struct mdc_chan *mchan);
 130         void (*disable_chan)(struct mdc_chan *mchan);
 131 };
 132
 133 struct mdc_dma {
 134         struct dma_device dma_dev;
 135         void __iomem *regs;
 136         struct clk *clk;
 137         struct dma_pool *desc_pool;
 138         struct regmap *periph_regs;
 139         spinlock_t lock;
 140         unsigned int nr_threads;
 141         unsigned int nr_channels;
 142         unsigned int bus_width;
 143         unsigned int max_burst_mult;
 144         unsigned int max_xfer_size;
 145         const struct mdc_dma_soc_data *soc;
 146         struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
 147 };
 148
 149 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
 150 {
 151         return readl(mdma->regs + reg);
 152 }
 153
 154 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
 155 {
 156         writel(val, mdma->regs + reg);
 157 }
 158
 159 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
 160 {
 161         return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
 162 }
 163
 164 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
 165 {
 166         mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
 167 }
 168
 169 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
 170 {
 171         return container_of(to_virt_chan(c), struct mdc_chan, vc);
 172 }
 173
 174 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
 175 {
 176         struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
 177
 178         return container_of(vdesc, struct mdc_tx_desc, vd);
 179 }
 180
 181 static inline struct device *mdma2dev(struct mdc_dma *mdma)
 182 {
 183         return mdma->dma_dev.dev;
 184 }
 185
 186 static inline unsigned int to_mdc_width(unsigned int bytes)
 187 {
 188         return ffs(bytes) - 1;
 189 }
 190
 191 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
 192                                       unsigned int bytes)
 193 {
 194         ldesc->gen_conf |= to_mdc_width(bytes) <<
 195                 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
 196 }
 197
 198 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
 199                                        unsigned int bytes)
 200 {
 201         ldesc->gen_conf |= to_mdc_width(bytes) <<
 202                 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
 203 }
 204
 205 static void mdc_list_desc_config(struct mdc_chan *mchan,
 206                                  struct mdc_hw_list_desc *ldesc,
 207                                  enum dma_transfer_direction dir,
 208                                  dma_addr_t src, dma_addr_t dst, size_t len)
 209 {
 210         struct mdc_dma *mdma = mchan->mdma;
 211         unsigned int max_burst, burst_size;
 212
 213         ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
 214                 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 215                 MDC_GENERAL_CONFIG_PHYSICAL_R;
 216         ldesc->readport_conf =
 217                 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 218                 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 219                 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 220         ldesc->read_addr = src;
 221         ldesc->write_addr = dst;
 222         ldesc->xfer_size = len - 1;
 223         ldesc->node_addr = 0;
 224         ldesc->cmds_done = 0;
 225         ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
 226                 MDC_CONTROL_AND_STATUS_EN;
 227         ldesc->next_desc = NULL;
 228
 229         if (IS_ALIGNED(dst, mdma->bus_width) &&
 230             IS_ALIGNED(src, mdma->bus_width))
 231                 max_burst = mdma->bus_width * mdma->max_burst_mult;
 232         else
 233                 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
 234
 235         if (dir == DMA_MEM_TO_DEV) {
 236                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
 237                 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 238                 mdc_set_read_width(ldesc, mdma->bus_width);
 239                 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
 240                 burst_size = min(max_burst, mchan->config.dst_maxburst *
 241                                  mchan->config.dst_addr_width);
 242         } else if (dir == DMA_DEV_TO_MEM) {
 243                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
 244                 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
 245                 mdc_set_read_width(ldesc, mchan->config.src_addr_width);
 246                 mdc_set_write_width(ldesc, mdma->bus_width);
 247                 burst_size = min(max_burst, mchan->config.src_maxburst *
 248                                  mchan->config.src_addr_width);
 249         } else {
 250                 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
 251                         MDC_GENERAL_CONFIG_INC_W;
 252                 mdc_set_read_width(ldesc, mdma->bus_width);
 253                 mdc_set_write_width(ldesc, mdma->bus_width);
 254                 burst_size = max_burst;
 255         }
 256         ldesc->readport_conf |= (burst_size - 1) <<
 257                 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
 258 }
 259
 260 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
 261 {
 262         struct mdc_dma *mdma = mdesc->chan->mdma;
 263         struct mdc_hw_list_desc *curr, *next;
 264         dma_addr_t curr_phys, next_phys;
 265
 266         curr = mdesc->list;
 267         curr_phys = mdesc->list_phys;
 268         while (curr) {
 269                 next = curr->next_desc;
 270                 next_phys = curr->node_addr;
 271                 dma_pool_free(mdma->desc_pool, curr, curr_phys);
 272                 curr = next;
 273                 curr_phys = next_phys;
 274         }
 275 }
 276
 277 static void mdc_desc_free(struct virt_dma_desc *vd)
 278 {
 279         struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
 280
 281         mdc_list_desc_free(mdesc);
 282         kfree(mdesc);
 283 }
 284
 285 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
 286         struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
 287         unsigned long flags)
 288 {
 289         struct mdc_chan *mchan = to_mdc_chan(chan);
 290         struct mdc_dma *mdma = mchan->mdma;
 291         struct mdc_tx_desc *mdesc;
 292         struct mdc_hw_list_desc *curr, *prev = NULL;
 293         dma_addr_t curr_phys;
 294
 295         if (!len)
 296                 return NULL;
 297
 298         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 299         if (!mdesc)
 300                 return NULL;
 301         mdesc->chan = mchan;
 302         mdesc->list_xfer_size = len;
 303
 304         while (len > 0) {
 305                 size_t xfer_size;
 306
 307                 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
 308                 if (!curr)
 309                         goto free_desc;
 310
 311                 if (prev) {
 312                         prev->node_addr = curr_phys;
 313                         prev->next_desc = curr;
 314                 } else {
 315                         mdesc->list_phys = curr_phys;
 316                         mdesc->list = curr;
 317                 }
 318
 319                 xfer_size = min_t(size_t, mdma->max_xfer_size, len);
 320
 321                 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
 322                                      xfer_size);
 323
 324                 prev = curr;
 325
 326                 mdesc->list_len++;
 327                 src += xfer_size;
 328                 dest += xfer_size;
 329                 len -= xfer_size;
 330         }
 331
 332         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 333
 334 free_desc:
 335         mdc_desc_free(&mdesc->vd);
 336
 337         return NULL;
 338 }
 339
 340 static int mdc_check_slave_width(struct mdc_chan *mchan,
 341                                  enum dma_transfer_direction dir)
 342 {
 343         enum dma_slave_buswidth width;
 344
 345         if (dir == DMA_MEM_TO_DEV)
 346                 width = mchan->config.dst_addr_width;
 347         else
 348                 width = mchan->config.src_addr_width;
 349
 350         switch (width) {
 351         case DMA_SLAVE_BUSWIDTH_1_BYTE:
 352         case DMA_SLAVE_BUSWIDTH_2_BYTES:
 353         case DMA_SLAVE_BUSWIDTH_4_BYTES:
 354         case DMA_SLAVE_BUSWIDTH_8_BYTES:
 355                 break;
 356         default:
 357                 return -EINVAL;
 358         }
 359
 360         if (width > mchan->mdma->bus_width)
 361                 return -EINVAL;
 362
 363         return 0;
 364 }
 365
 366 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
 367         struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
 368         size_t period_len, enum dma_transfer_direction dir,
 369         unsigned long flags)
 370 {
 371         struct mdc_chan *mchan = to_mdc_chan(chan);
 372         struct mdc_dma *mdma = mchan->mdma;
 373         struct mdc_tx_desc *mdesc;
 374         struct mdc_hw_list_desc *curr, *prev = NULL;
 375         dma_addr_t curr_phys;
 376
 377         if (!buf_len && !period_len)
 378                 return NULL;
 379
 380         if (!is_slave_direction(dir))
 381                 return NULL;
 382
 383         if (mdc_check_slave_width(mchan, dir) < 0)
 384                 return NULL;
 385
 386         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 387         if (!mdesc)
 388                 return NULL;
 389         mdesc->chan = mchan;
 390         mdesc->cyclic = true;
 391         mdesc->list_xfer_size = buf_len;
 392         mdesc->list_period_len = DIV_ROUND_UP(period_len,
 393                                               mdma->max_xfer_size);
 394
 395         while (buf_len > 0) {
 396                 size_t remainder = min(period_len, buf_len);
 397
 398                 while (remainder > 0) {
 399                         size_t xfer_size;
 400
 401                         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 402                                               &curr_phys);
 403                         if (!curr)
 404                                 goto free_desc;
 405
 406                         if (!prev) {
 407                                 mdesc->list_phys = curr_phys;
 408                                 mdesc->list = curr;
 409                         } else {
 410                                 prev->node_addr = curr_phys;
 411                                 prev->next_desc = curr;
 412                         }
 413
 414                         xfer_size = min_t(size_t, mdma->max_xfer_size,
 415                                           remainder);
 416
 417                         if (dir == DMA_MEM_TO_DEV) {
 418                                 mdc_list_desc_config(mchan, curr, dir,
 419                                                      buf_addr,
 420                                                      mchan->config.dst_addr,
 421                                                      xfer_size);
 422                         } else {
 423                                 mdc_list_desc_config(mchan, curr, dir,
 424                                                      mchan->config.src_addr,
 425                                                      buf_addr,
 426                                                      xfer_size);
 427                         }
 428
 429                         prev = curr;
 430
 431                         mdesc->list_len++;
 432                         buf_addr += xfer_size;
 433                         buf_len -= xfer_size;
 434                         remainder -= xfer_size;
 435                 }
 436         }
 437         prev->node_addr = mdesc->list_phys;
 438
 439         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 440
 441 free_desc:
 442         mdc_desc_free(&mdesc->vd);
 443
 444         return NULL;
 445 }
 446
 447 static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
 448         struct dma_chan *chan, struct scatterlist *sgl,
 449         unsigned int sg_len, enum dma_transfer_direction dir,
 450         unsigned long flags, void *context)
 451 {
 452         struct mdc_chan *mchan = to_mdc_chan(chan);
 453         struct mdc_dma *mdma = mchan->mdma;
 454         struct mdc_tx_desc *mdesc;
 455         struct scatterlist *sg;
 456         struct mdc_hw_list_desc *curr, *prev = NULL;
 457         dma_addr_t curr_phys;
 458         unsigned int i;
 459
 460         if (!sgl)
 461                 return NULL;
 462
 463         if (!is_slave_direction(dir))
 464                 return NULL;
 465
 466         if (mdc_check_slave_width(mchan, dir) < 0)
 467                 return NULL;
 468
 469         mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
 470         if (!mdesc)
 471                 return NULL;
 472         mdesc->chan = mchan;
 473
 474         for_each_sg(sgl, sg, sg_len, i) {
 475                 dma_addr_t buf = sg_dma_address(sg);
 476                 size_t buf_len = sg_dma_len(sg);
 477
 478                 while (buf_len > 0) {
 479                         size_t xfer_size;
 480
 481                         curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
 482                                               &curr_phys);
 483                         if (!curr)
 484                                 goto free_desc;
 485
 486                         if (!prev) {
 487                                 mdesc->list_phys = curr_phys;
 488                                 mdesc->list = curr;
 489                         } else {
 490                                 prev->node_addr = curr_phys;
 491                                 prev->next_desc = curr;
 492                         }
 493
 494                         xfer_size = min_t(size_t, mdma->max_xfer_size,
 495                                           buf_len);
 496
 497                         if (dir == DMA_MEM_TO_DEV) {
 498                                 mdc_list_desc_config(mchan, curr, dir, buf,
 499                                                      mchan->config.dst_addr,
 500                                                      xfer_size);
 501                         } else {
 502                                 mdc_list_desc_config(mchan, curr, dir,
 503                                                      mchan->config.src_addr,
 504                                                      buf, xfer_size);
 505                         }
 506
 507                         prev = curr;
 508
 509                         mdesc->list_len++;
 510                         mdesc->list_xfer_size += xfer_size;
 511                         buf += xfer_size;
 512                         buf_len -= xfer_size;
 513                 }
 514         }
 515
 516         return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
 517
 518 free_desc:
 519         mdc_desc_free(&mdesc->vd);
 520
 521         return NULL;
 522 }
 523
 524 static void mdc_issue_desc(struct mdc_chan *mchan)
 525 {
 526         struct mdc_dma *mdma = mchan->mdma;
 527         struct virt_dma_desc *vd;
 528         struct mdc_tx_desc *mdesc;
 529         u32 val;
 530
 531         vd = vchan_next_desc(&mchan->vc);
 532         if (!vd)
 533                 return;
 534
 535         list_del(&vd->node);
 536
 537         mdesc = to_mdc_desc(&vd->tx);
 538         mchan->desc = mdesc;
 539
 540         dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
 541                 mchan->chan_nr);
 542
 543         mdma->soc->enable_chan(mchan);
 544
 545         val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
 546         val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
 547                 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
 548                 MDC_GENERAL_CONFIG_PHYSICAL_R;
 549         mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
 550         val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
 551                 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
 552                 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
 553         mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
 554         mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
 555         val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
 556         val |= MDC_CONTROL_AND_STATUS_LIST_EN;
 557         mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
 558 }
 559
 560 static void mdc_issue_pending(struct dma_chan *chan)
 561 {
 562         struct mdc_chan *mchan = to_mdc_chan(chan);
 563         unsigned long flags;
 564
 565         spin_lock_irqsave(&mchan->vc.lock, flags);
 566         if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
 567                 mdc_issue_desc(mchan);
 568         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 569 }
 570
 571 static enum dma_status mdc_tx_status(struct dma_chan *chan,
 572         dma_cookie_t cookie, struct dma_tx_state *txstate)
 573 {
 574         struct mdc_chan *mchan = to_mdc_chan(chan);
 575         struct mdc_tx_desc *mdesc;
 576         struct virt_dma_desc *vd;
 577         unsigned long flags;
 578         size_t bytes = 0;
 579         int ret;
 580
 581         ret = dma_cookie_status(chan, cookie, txstate);
 582         if (ret == DMA_COMPLETE)
 583                 return ret;
 584
 585         if (!txstate)
 586                 return ret;
 587
 588         spin_lock_irqsave(&mchan->vc.lock, flags);
 589         vd = vchan_find_desc(&mchan->vc, cookie);
 590         if (vd) {
 591                 mdesc = to_mdc_desc(&vd->tx);
 592                 bytes = mdesc->list_xfer_size;
 593         } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
 594                 struct mdc_hw_list_desc *ldesc;
 595                 u32 val1, val2, done, processed, residue;
 596                 int i, cmds;
 597
 598                 mdesc = mchan->desc;
 599
 600                 /*
 601                  * Determine the number of commands that haven't been
 602                  * processed (handled by the IRQ handler) yet.
 603                  */
 604                 do {
 605                         val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 606                                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 607                         residue = mdc_chan_readl(mchan,
 608                                                  MDC_ACTIVE_TRANSFER_SIZE);
 609                         val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
 610                                 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
 611                 } while (val1 != val2);
 612
 613                 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 614                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 615                 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 616                         MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 617                 cmds = (done - processed) %
 618                         (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
 619
 620                 /*
 621                  * If the command loaded event hasn't been processed yet, then
 622                  * the difference above includes an extra command.
 623                  */
 624                 if (!mdesc->cmd_loaded)
 625                         cmds--;
 626                 else
 627                         cmds += mdesc->list_cmds_done;
 628
 629                 bytes = mdesc->list_xfer_size;
 630                 ldesc = mdesc->list;
 631                 for (i = 0; i < cmds; i++) {
 632                         bytes -= ldesc->xfer_size + 1;
 633                         ldesc = ldesc->next_desc;
 634                 }
 635                 if (ldesc) {
 636                         if (residue != MDC_TRANSFER_SIZE_MASK)
 637                                 bytes -= ldesc->xfer_size - residue;
 638                         else
 639                                 bytes -= ldesc->xfer_size + 1;
 640                 }
 641         }
 642         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 643
 644         dma_set_residue(txstate, bytes);
 645
 646         return ret;
 647 }
 648
 649 static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
 650 {
 651         u32 val, processed, done1, done2;
 652         unsigned int ret;
 653
 654         val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 655         processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
 656                                 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
 657         /*
 658          * CMDS_DONE may have incremented between reading CMDS_PROCESSED
 659          * and clearing INT_ACTIVE.  Re-read CMDS_PROCESSED to ensure we
 660          * didn't miss a command completion.
 661          */
 662         do {
 663                 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 664
 665                 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 666                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 667
 668                 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
 669                           MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
 670                          MDC_CMDS_PROCESSED_INT_ACTIVE);
 671
 672                 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
 673
 674                 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
 675
 676                 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
 677
 678                 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
 679                         MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
 680         } while (done1 != done2);
 681
 682         if (done1 >= processed)
 683                 ret = done1 - processed;
 684         else
 685                 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
 686                         processed) + done1;
 687
 688         return ret;
 689 }
 690
 691 static int mdc_terminate_all(struct dma_chan *chan)
 692 {
 693         struct mdc_chan *mchan = to_mdc_chan(chan);
 694         unsigned long flags;
 695         LIST_HEAD(head);
 696
 697         spin_lock_irqsave(&mchan->vc.lock, flags);
 698
 699         mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
 700                         MDC_CONTROL_AND_STATUS);
 701
 702         if (mchan->desc) {
 703                 vchan_terminate_vdesc(&mchan->desc->vd);
 704                 mchan->desc = NULL;
 705         }
 706         vchan_get_all_descriptors(&mchan->vc, &head);
 707
 708         mdc_get_new_events(mchan);
 709
 710         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 711
 712         vchan_dma_desc_free_list(&mchan->vc, &head);
 713
 714         return 0;
 715 }
 716
 717 static void mdc_synchronize(struct dma_chan *chan)
 718 {
 719         struct mdc_chan *mchan = to_mdc_chan(chan);
 720
 721         vchan_synchronize(&mchan->vc);
 722 }
 723
 724 static int mdc_slave_config(struct dma_chan *chan,
 725                             struct dma_slave_config *config)
 726 {
 727         struct mdc_chan *mchan = to_mdc_chan(chan);
 728         unsigned long flags;
 729
 730         spin_lock_irqsave(&mchan->vc.lock, flags);
 731         mchan->config = *config;
 732         spin_unlock_irqrestore(&mchan->vc.lock, flags);
 733
 734         return 0;
 735 }
 736
 737 static int mdc_alloc_chan_resources(struct dma_chan *chan)
 738 {
 739         struct mdc_chan *mchan = to_mdc_chan(chan);
 740         struct device *dev = mdma2dev(mchan->mdma);
 741
 742         return pm_runtime_get_sync(dev);
 743 }
 744
 745 static void mdc_free_chan_resources(struct dma_chan *chan)
 746 {
 747         struct mdc_chan *mchan = to_mdc_chan(chan);
 748         struct mdc_dma *mdma = mchan->mdma;
 749         struct device *dev = mdma2dev(mdma);
 750
 751         mdc_terminate_all(chan);
 752         mdma->soc->disable_chan(mchan);
 753         pm_runtime_put(dev);
 754 }
 755
 756 static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
 757 {
 758         struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
 759         struct mdc_tx_desc *mdesc;
 760         unsigned int i, new_events;
 761
 762         spin_lock(&mchan->vc.lock);
 763
 764         dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
 765
 766         new_events = mdc_get_new_events(mchan);
 767
 768         if (!new_events)
 769                 goto out;
 770
 771         mdesc = mchan->desc;
 772         if (!mdesc) {
 773                 dev_warn(mdma2dev(mchan->mdma),
 774                          "IRQ with no active descriptor on channel %d\n",
 775                          mchan->chan_nr);
 776                 goto out;
 777         }
 778
 779         for (i = 0; i < new_events; i++) {
 780                 /*
 781                  * The first interrupt in a transfer indicates that the
 782                  * command list has been loaded, not that a command has
 783                  * been completed.
 784                  */
 785                 if (!mdesc->cmd_loaded) {
 786                         mdesc->cmd_loaded = true;
 787                         continue;
 788                 }
 789
 790                 mdesc->list_cmds_done++;
 791                 if (mdesc->cyclic) {
 792                         mdesc->list_cmds_done %= mdesc->list_len;
 793                         if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
 794                                 vchan_cyclic_callback(&mdesc->vd);
 795                 } else if (mdesc->list_cmds_done == mdesc->list_len) {
 796                         mchan->desc = NULL;
 797                         vchan_cookie_complete(&mdesc->vd);
 798                         mdc_issue_desc(mchan);
 799                         break;
 800                 }
 801         }
 802 out:
 803         spin_unlock(&mchan->vc.lock);
 804
 805         return IRQ_HANDLED;
 806 }
 807
 808 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
 809                                      struct of_dma *ofdma)
 810 {
 811         struct mdc_dma *mdma = ofdma->of_dma_data;
 812         struct dma_chan *chan;
 813
 814         if (dma_spec->args_count != 3)
 815                 return NULL;
 816
 817         list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
 818                 struct mdc_chan *mchan = to_mdc_chan(chan);
 819
 820                 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
 821                         continue;
 822                 if (dma_get_slave_channel(chan)) {
 823                         mchan->periph = dma_spec->args[0];
 824                         mchan->thread = dma_spec->args[2];
 825                         return chan;
 826                 }
 827         }
 828
 829         return NULL;
 830 }
 831
 832 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch)       (0x120 + 0x4 * ((ch) / 4))
 833 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
 834 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK      0x3f
 835
 836 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
 837 {
 838         struct mdc_dma *mdma = mchan->mdma;
 839
 840         regmap_update_bits(mdma->periph_regs,
 841                            PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 842                            PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 843                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 844                            mchan->periph <<
 845                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
 846 }
 847
 848 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
 849 {
 850         struct mdc_dma *mdma = mchan->mdma;
 851
 852         regmap_update_bits(mdma->periph_regs,
 853                            PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
 854                            PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
 855                            PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
 856                            0);
 857 }
 858
 859 static const struct mdc_dma_soc_data pistachio_mdc_data = {
 860         .enable_chan = pistachio_mdc_enable_chan,
 861         .disable_chan = pistachio_mdc_disable_chan,
 862 };
 863
 864 static const struct of_device_id mdc_dma_of_match[] = {
 865         { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
 866         { },
 867 };
 868 MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
 869
 870 static int img_mdc_runtime_suspend(struct device *dev)
 871 {
 872         struct mdc_dma *mdma = dev_get_drvdata(dev);
 873
 874         clk_disable_unprepare(mdma->clk);
 875
 876         return 0;
 877 }
 878
 879 static int img_mdc_runtime_resume(struct device *dev)
 880 {
 881         struct mdc_dma *mdma = dev_get_drvdata(dev);
 882
 883         return clk_prepare_enable(mdma->clk);
 884 }
 885
 886 static int mdc_dma_probe(struct platform_device *pdev)
 887 {
 888         struct mdc_dma *mdma;
 889         struct resource *res;
 890         unsigned int i;
 891         u32 val;
 892         int ret;
 893
 894         mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
 895         if (!mdma)
 896                 return -ENOMEM;
 897         platform_set_drvdata(pdev, mdma);
 898
 899         mdma->soc = of_device_get_match_data(&pdev->dev);
 900
 901         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 902         mdma->regs = devm_ioremap_resource(&pdev->dev, res);
 903         if (IS_ERR(mdma->regs))
 904                 return PTR_ERR(mdma->regs);
 905
 906         mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 907                                                             "img,cr-periph");
 908         if (IS_ERR(mdma->periph_regs))
 909                 return PTR_ERR(mdma->periph_regs);
 910
 911         mdma->clk = devm_clk_get(&pdev->dev, "sys");
 912         if (IS_ERR(mdma->clk))
 913                 return PTR_ERR(mdma->clk);
 914
 915         dma_cap_zero(mdma->dma_dev.cap_mask);
 916         dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
 917         dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
 918         dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
 919         dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
 920
 921         val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
 922         mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
 923                 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
 924         mdma->nr_threads =
 925                 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
 926                       MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
 927         mdma->bus_width =
 928                 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
 929                        MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
 930         /*
 931          * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
 932          * are supported, this makes it possible for the value reported in
 933          * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
 934          * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
 935          * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining.  To eliminate this
 936          * ambiguity, restrict transfer sizes to one bus-width less than the
 937          * actual maximum.
 938          */
 939         mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
 940
 941         of_property_read_u32(pdev->dev.of_node, "dma-channels",
 942                              &mdma->nr_channels);
 943         ret = of_property_read_u32(pdev->dev.of_node,
 944                                    "img,max-burst-multiplier",
 945                                    &mdma->max_burst_mult);
 946         if (ret)
 947                 return ret;
 948
 949         mdma->dma_dev.dev = &pdev->dev;
 950         mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
 951         mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
 952         mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
 953         mdma->dma_dev.device_alloc_chan_resources = mdc_alloc_chan_resources;
 954         mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
 955         mdma->dma_dev.device_tx_status = mdc_tx_status;
 956         mdma->dma_dev.device_issue_pending = mdc_issue_pending;
 957         mdma->dma_dev.device_terminate_all = mdc_terminate_all;
 958         mdma->dma_dev.device_synchronize = mdc_synchronize;
 959         mdma->dma_dev.device_config = mdc_slave_config;
 960
 961         mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 962         mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 963         for (i = 1; i <= mdma->bus_width; i <<= 1) {
 964                 mdma->dma_dev.src_addr_widths |= BIT(i);
 965                 mdma->dma_dev.dst_addr_widths |= BIT(i);
 966         }
 967
 968         INIT_LIST_HEAD(&mdma->dma_dev.channels);
 969         for (i = 0; i < mdma->nr_channels; i++) {
 970                 struct mdc_chan *mchan = &mdma->channels[i];
 971
 972                 mchan->mdma = mdma;
 973                 mchan->chan_nr = i;
 974                 mchan->irq = platform_get_irq(pdev, i);
 975                 if (mchan->irq < 0)
 976                         return mchan->irq;
 977
 978                 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
 979                                        IRQ_TYPE_LEVEL_HIGH,
 980                                        dev_name(&pdev->dev), mchan);
 981                 if (ret < 0)
 982                         return ret;
 983
 984                 mchan->vc.desc_free = mdc_desc_free;
 985                 vchan_init(&mchan->vc, &mdma->dma_dev);
 986         }
 987
 988         mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
 989                                            sizeof(struct mdc_hw_list_desc),
 990                                            4, 0);
 991         if (!mdma->desc_pool)
 992                 return -ENOMEM;
 993
 994         pm_runtime_enable(&pdev->dev);
 995         if (!pm_runtime_enabled(&pdev->dev)) {
 996                 ret = img_mdc_runtime_resume(&pdev->dev);
 997                 if (ret)
 998                         return ret;
 999         }
1000
1001         ret = dma_async_device_register(&mdma->dma_dev);
1002         if (ret)
1003                 goto suspend;
1004
1005         ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
1006         if (ret)
1007                 goto unregister;
1008
1009         dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
1010                  mdma->nr_channels, mdma->nr_threads);
1011
1012         return 0;
1013
1014 unregister:
1015         dma_async_device_unregister(&mdma->dma_dev);
1016 suspend:
1017         if (!pm_runtime_enabled(&pdev->dev))
1018                 img_mdc_runtime_suspend(&pdev->dev);
1019         pm_runtime_disable(&pdev->dev);
1020         return ret;
1021 }
1022
1023 static int mdc_dma_remove(struct platform_device *pdev)
1024 {
1025         struct mdc_dma *mdma = platform_get_drvdata(pdev);
1026         struct mdc_chan *mchan, *next;
1027
1028         of_dma_controller_free(pdev->dev.of_node);
1029         dma_async_device_unregister(&mdma->dma_dev);
1030
1031         list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1032                                  vc.chan.device_node) {
1033                 list_del(&mchan->vc.chan.device_node);
1034
1035                 devm_free_irq(&pdev->dev, mchan->irq, mchan);
1036
1037                 tasklet_kill(&mchan->vc.task);
1038         }
1039
1040         pm_runtime_disable(&pdev->dev);
1041         if (!pm_runtime_status_suspended(&pdev->dev))
1042                 img_mdc_runtime_suspend(&pdev->dev);
1043
1044         return 0;
1045 }
1046
1047 #ifdef CONFIG_PM_SLEEP
1048 static int img_mdc_suspend_late(struct device *dev)
1049 {
1050         struct mdc_dma *mdma = dev_get_drvdata(dev);
1051         int i;
1052
1053         /* Check that all channels are idle */
1054         for (i = 0; i < mdma->nr_channels; i++) {
1055                 struct mdc_chan *mchan = &mdma->channels[i];
1056
1057                 if (unlikely(mchan->desc))
1058                         return -EBUSY;
1059         }
1060
1061         return pm_runtime_force_suspend(dev);
1062 }
1063
1064 static int img_mdc_resume_early(struct device *dev)
1065 {
1066         return pm_runtime_force_resume(dev);
1067 }
1068 #endif /* CONFIG_PM_SLEEP */
1069
1070 static const struct dev_pm_ops img_mdc_pm_ops = {
1071         SET_RUNTIME_PM_OPS(img_mdc_runtime_suspend,
1072                            img_mdc_runtime_resume, NULL)
1073         SET_LATE_SYSTEM_SLEEP_PM_OPS(img_mdc_suspend_late,
1074                                      img_mdc_resume_early)
1075 };
1076
1077 static struct platform_driver mdc_dma_driver = {
1078         .driver = {
1079                 .name = "img-mdc-dma",
1080                 .pm = &img_mdc_pm_ops,
1081                 .of_match_table = of_match_ptr(mdc_dma_of_match),
1082         },
1083         .probe = mdc_dma_probe,
1084         .remove = mdc_dma_remove,
1085 };
1086 module_platform_driver(mdc_dma_driver);
1087
1088 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1089 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1090 MODULE_LICENSE("GPL v2");