2 * TI EDMA DMA engine driver
4 * Copyright 2012 Texas Instruments
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <linux/dmaengine.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/interrupt.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/slab.h>
25 #include <linux/spinlock.h>
27 #include <linux/platform_data/edma.h>
29 #include "dmaengine.h"
33 * This will go away when the private EDMA API is folded
34 * into this driver and the platform device(s) are
35 * instantiated in the arch code. We can only get away
36 * with this simplification because DA8XX may not be built
37 * in the same kernel image with other DaVinci parts. This
38 * avoids having to sprinkle dmaengine driver platform devices
39 * and data throughout all the existing board files.
41 #ifdef CONFIG_ARCH_DAVINCI_DA8XX
47 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */
49 /* Max of 16 segments per channel to conserve PaRAM slots */
51 #define EDMA_MAX_SLOTS MAX_NR_SG
52 #define EDMA_DESCRIPTORS 16
55 struct virt_dma_desc vdesc
;
56 struct list_head node
;
60 struct edmacc_param pset
[0];
66 struct virt_dma_chan vchan
;
67 struct list_head node
;
68 struct edma_desc
*edesc
;
72 int slot
[EDMA_MAX_SLOTS
];
74 struct dma_slave_config cfg
;
79 struct dma_device dma_slave
;
80 struct edma_chan slave_chans
[EDMA_CHANS
];
85 static inline struct edma_cc
*to_edma_cc(struct dma_device
*d
)
87 return container_of(d
, struct edma_cc
, dma_slave
);
90 static inline struct edma_chan
*to_edma_chan(struct dma_chan
*c
)
92 return container_of(c
, struct edma_chan
, vchan
.chan
);
95 static inline struct edma_desc
96 *to_edma_desc(struct dma_async_tx_descriptor
*tx
)
98 return container_of(tx
, struct edma_desc
, vdesc
.tx
);
101 static void edma_desc_free(struct virt_dma_desc
*vdesc
)
103 kfree(container_of(vdesc
, struct edma_desc
, vdesc
));
106 /* Dispatch a queued descriptor to the controller (caller holds lock) */
107 static void edma_execute(struct edma_chan
*echan
)
109 struct virt_dma_desc
*vdesc
;
110 struct edma_desc
*edesc
;
111 struct device
*dev
= echan
->vchan
.chan
.device
->dev
;
112 int i
, j
, left
, nslots
;
114 /* If either we processed all psets or we're still not started */
116 echan
->edesc
->pset_nr
== echan
->edesc
->processed
) {
118 vdesc
= vchan_next_desc(&echan
->vchan
);
123 list_del(&vdesc
->node
);
124 echan
->edesc
= to_edma_desc(&vdesc
->tx
);
127 edesc
= echan
->edesc
;
129 /* Find out how many left */
130 left
= edesc
->pset_nr
- edesc
->processed
;
131 nslots
= min(MAX_NR_SG
, left
);
133 /* Write descriptor PaRAM set(s) */
134 for (i
= 0; i
< nslots
; i
++) {
135 j
= i
+ edesc
->processed
;
136 edma_write_slot(echan
->slot
[i
], &edesc
->pset
[j
]);
137 dev_dbg(echan
->vchan
.chan
.device
->dev
,
149 j
, echan
->ch_num
, echan
->slot
[i
],
153 edesc
->pset
[j
].a_b_cnt
,
155 edesc
->pset
[j
].src_dst_bidx
,
156 edesc
->pset
[j
].src_dst_cidx
,
157 edesc
->pset
[j
].link_bcntrld
);
158 /* Link to the previous slot if not the last set */
159 if (i
!= (nslots
- 1))
160 edma_link(echan
->slot
[i
], echan
->slot
[i
+1]);
163 edesc
->processed
+= nslots
;
166 * If this is either the last set in a set of SG-list transactions
167 * then setup a link to the dummy slot, this results in all future
168 * events being absorbed and that's OK because we're done
170 if (edesc
->processed
== edesc
->pset_nr
)
171 edma_link(echan
->slot
[nslots
-1], echan
->ecc
->dummy_slot
);
173 if (edesc
->processed
<= MAX_NR_SG
) {
174 dev_dbg(dev
, "first transfer starting %d\n", echan
->ch_num
);
175 edma_start(echan
->ch_num
);
177 dev_dbg(dev
, "chan: %d: completed %d elements, resuming\n",
178 echan
->ch_num
, edesc
->processed
);
179 edma_resume(echan
->ch_num
);
183 * This happens due to setup times between intermediate transfers
184 * in long SG lists which have to be broken up into transfers of
188 dev_dbg(dev
, "missed event in execute detected\n");
189 edma_clean_channel(echan
->ch_num
);
190 edma_stop(echan
->ch_num
);
191 edma_start(echan
->ch_num
);
192 edma_trigger_channel(echan
->ch_num
);
197 static int edma_terminate_all(struct edma_chan
*echan
)
202 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
205 * Stop DMA activity: we assume the callback will not be called
206 * after edma_dma() returns (even if it does, it will see
207 * echan->edesc is NULL and exit.)
211 edma_stop(echan
->ch_num
);
214 vchan_get_all_descriptors(&echan
->vchan
, &head
);
215 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
216 vchan_dma_desc_free_list(&echan
->vchan
, &head
);
221 static int edma_slave_config(struct edma_chan
*echan
,
222 struct dma_slave_config
*cfg
)
224 if (cfg
->src_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
||
225 cfg
->dst_addr_width
== DMA_SLAVE_BUSWIDTH_8_BYTES
)
228 memcpy(&echan
->cfg
, cfg
, sizeof(echan
->cfg
));
233 static int edma_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
237 struct dma_slave_config
*config
;
238 struct edma_chan
*echan
= to_edma_chan(chan
);
241 case DMA_TERMINATE_ALL
:
242 edma_terminate_all(echan
);
244 case DMA_SLAVE_CONFIG
:
245 config
= (struct dma_slave_config
*)arg
;
246 ret
= edma_slave_config(echan
, config
);
255 static struct dma_async_tx_descriptor
*edma_prep_slave_sg(
256 struct dma_chan
*chan
, struct scatterlist
*sgl
,
257 unsigned int sg_len
, enum dma_transfer_direction direction
,
258 unsigned long tx_flags
, void *context
)
260 struct edma_chan
*echan
= to_edma_chan(chan
);
261 struct device
*dev
= chan
->device
->dev
;
262 struct edma_desc
*edesc
;
264 enum dma_slave_buswidth dev_width
;
266 struct scatterlist
*sg
;
267 int acnt
, bcnt
, ccnt
, src
, dst
, cidx
;
268 int src_bidx
, dst_bidx
, src_cidx
, dst_cidx
;
271 if (unlikely(!echan
|| !sgl
|| !sg_len
))
274 if (direction
== DMA_DEV_TO_MEM
) {
275 dev_addr
= echan
->cfg
.src_addr
;
276 dev_width
= echan
->cfg
.src_addr_width
;
277 burst
= echan
->cfg
.src_maxburst
;
278 } else if (direction
== DMA_MEM_TO_DEV
) {
279 dev_addr
= echan
->cfg
.dst_addr
;
280 dev_width
= echan
->cfg
.dst_addr_width
;
281 burst
= echan
->cfg
.dst_maxburst
;
283 dev_err(dev
, "%s: bad direction?\n", __func__
);
287 if (dev_width
== DMA_SLAVE_BUSWIDTH_UNDEFINED
) {
288 dev_err(dev
, "Undefined slave buswidth\n");
292 edesc
= kzalloc(sizeof(*edesc
) + sg_len
*
293 sizeof(edesc
->pset
[0]), GFP_ATOMIC
);
295 dev_dbg(dev
, "Failed to allocate a descriptor\n");
299 edesc
->pset_nr
= sg_len
;
301 /* Allocate a PaRAM slot, if needed */
302 nslots
= min_t(unsigned, MAX_NR_SG
, sg_len
);
304 for (i
= 0; i
< nslots
; i
++) {
305 if (echan
->slot
[i
] < 0) {
307 edma_alloc_slot(EDMA_CTLR(echan
->ch_num
),
309 if (echan
->slot
[i
] < 0) {
311 dev_err(dev
, "Failed to allocate slot\n");
318 /* Configure PaRAM sets for each SG */
319 for_each_sg(sgl
, sg
, sg_len
, i
) {
324 * If the maxburst is equal to the fifo width, use
325 * A-synced transfers. This allows for large contiguous
326 * buffer transfers using only one PaRAM set.
329 edesc
->absync
= false;
330 ccnt
= sg_dma_len(sg
) / acnt
/ (SZ_64K
- 1);
331 bcnt
= sg_dma_len(sg
) / acnt
- ccnt
* (SZ_64K
- 1);
338 * If maxburst is greater than the fifo address_width,
339 * use AB-synced transfers where A count is the fifo
340 * address_width and B count is the maxburst. In this
341 * case, we are limited to transfers of C count frames
342 * of (address_width * maxburst) where C count is limited
343 * to SZ_64K-1. This places an upper bound on the length
344 * of an SG segment that can be handled.
347 edesc
->absync
= true;
349 ccnt
= sg_dma_len(sg
) / (acnt
* bcnt
);
350 if (ccnt
> (SZ_64K
- 1)) {
351 dev_err(dev
, "Exceeded max SG segment size\n");
358 if (direction
== DMA_MEM_TO_DEV
) {
359 src
= sg_dma_address(sg
);
367 dst
= sg_dma_address(sg
);
374 edesc
->pset
[i
].opt
= EDMA_TCC(EDMA_CHAN_SLOT(echan
->ch_num
));
375 /* Configure A or AB synchronized transfers */
377 edesc
->pset
[i
].opt
|= SYNCDIM
;
379 /* If this is the last in a current SG set of transactions,
380 enable interrupts so that next set is processed */
381 if (!((i
+1) % MAX_NR_SG
))
382 edesc
->pset
[i
].opt
|= TCINTEN
;
384 /* If this is the last set, enable completion interrupt flag */
386 edesc
->pset
[i
].opt
|= TCINTEN
;
388 edesc
->pset
[i
].src
= src
;
389 edesc
->pset
[i
].dst
= dst
;
391 edesc
->pset
[i
].src_dst_bidx
= (dst_bidx
<< 16) | src_bidx
;
392 edesc
->pset
[i
].src_dst_cidx
= (dst_cidx
<< 16) | src_cidx
;
394 edesc
->pset
[i
].a_b_cnt
= bcnt
<< 16 | acnt
;
395 edesc
->pset
[i
].ccnt
= ccnt
;
396 edesc
->pset
[i
].link_bcntrld
= 0xffffffff;
400 return vchan_tx_prep(&echan
->vchan
, &edesc
->vdesc
, tx_flags
);
403 static void edma_callback(unsigned ch_num
, u16 ch_status
, void *data
)
405 struct edma_chan
*echan
= data
;
406 struct device
*dev
= echan
->vchan
.chan
.device
->dev
;
407 struct edma_desc
*edesc
;
409 struct edmacc_param p
;
411 /* Pause the channel */
412 edma_pause(echan
->ch_num
);
416 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
418 edesc
= echan
->edesc
;
420 if (edesc
->processed
== edesc
->pset_nr
) {
421 dev_dbg(dev
, "Transfer complete, stopping channel %d\n", ch_num
);
422 edma_stop(echan
->ch_num
);
423 vchan_cookie_complete(&edesc
->vdesc
);
425 dev_dbg(dev
, "Intermediate transfer complete on channel %d\n", ch_num
);
431 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
435 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
437 edma_read_slot(EDMA_CHAN_SLOT(echan
->slot
[0]), &p
);
440 * Issue later based on missed flag which will be sure
442 * (1) we finished transmitting an intermediate slot and
443 * edma_execute is coming up.
444 * (2) or we finished current transfer and issue will
447 * Important note: issuing can be dangerous here and
448 * lead to some nasty recursion when we are in a NULL
449 * slot. So we avoid doing so and set the missed flag.
451 if (p
.a_b_cnt
== 0 && p
.ccnt
== 0) {
452 dev_dbg(dev
, "Error occurred, looks like slot is null, just setting miss\n");
456 * The slot is already programmed but the event got
457 * missed, so its safe to issue it here.
459 dev_dbg(dev
, "Error occurred but slot is non-null, TRIGGERING\n");
460 edma_clean_channel(echan
->ch_num
);
461 edma_stop(echan
->ch_num
);
462 edma_start(echan
->ch_num
);
463 edma_trigger_channel(echan
->ch_num
);
466 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
474 /* Alloc channel resources */
475 static int edma_alloc_chan_resources(struct dma_chan
*chan
)
477 struct edma_chan
*echan
= to_edma_chan(chan
);
478 struct device
*dev
= chan
->device
->dev
;
483 a_ch_num
= edma_alloc_channel(echan
->ch_num
, edma_callback
,
484 chan
, EVENTQ_DEFAULT
);
491 if (a_ch_num
!= echan
->ch_num
) {
492 dev_err(dev
, "failed to allocate requested channel %u:%u\n",
493 EDMA_CTLR(echan
->ch_num
),
494 EDMA_CHAN_SLOT(echan
->ch_num
));
499 echan
->alloced
= true;
500 echan
->slot
[0] = echan
->ch_num
;
502 dev_info(dev
, "allocated channel for %u:%u\n",
503 EDMA_CTLR(echan
->ch_num
), EDMA_CHAN_SLOT(echan
->ch_num
));
508 edma_free_channel(a_ch_num
);
513 /* Free channel resources */
514 static void edma_free_chan_resources(struct dma_chan
*chan
)
516 struct edma_chan
*echan
= to_edma_chan(chan
);
517 struct device
*dev
= chan
->device
->dev
;
520 /* Terminate transfers */
521 edma_stop(echan
->ch_num
);
523 vchan_free_chan_resources(&echan
->vchan
);
525 /* Free EDMA PaRAM slots */
526 for (i
= 1; i
< EDMA_MAX_SLOTS
; i
++) {
527 if (echan
->slot
[i
] >= 0) {
528 edma_free_slot(echan
->slot
[i
]);
533 /* Free EDMA channel */
534 if (echan
->alloced
) {
535 edma_free_channel(echan
->ch_num
);
536 echan
->alloced
= false;
539 dev_info(dev
, "freeing channel for %u\n", echan
->ch_num
);
542 /* Send pending descriptor to hardware */
543 static void edma_issue_pending(struct dma_chan
*chan
)
545 struct edma_chan
*echan
= to_edma_chan(chan
);
548 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
549 if (vchan_issue_pending(&echan
->vchan
) && !echan
->edesc
)
551 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
554 static size_t edma_desc_size(struct edma_desc
*edesc
)
560 for (size
= i
= 0; i
< edesc
->pset_nr
; i
++)
561 size
+= (edesc
->pset
[i
].a_b_cnt
& 0xffff) *
562 (edesc
->pset
[i
].a_b_cnt
>> 16) *
565 size
= (edesc
->pset
[0].a_b_cnt
& 0xffff) *
566 (edesc
->pset
[0].a_b_cnt
>> 16) +
567 (edesc
->pset
[0].a_b_cnt
& 0xffff) *
568 (SZ_64K
- 1) * edesc
->pset
[0].ccnt
;
573 /* Check request completion status */
574 static enum dma_status
edma_tx_status(struct dma_chan
*chan
,
576 struct dma_tx_state
*txstate
)
578 struct edma_chan
*echan
= to_edma_chan(chan
);
579 struct virt_dma_desc
*vdesc
;
583 ret
= dma_cookie_status(chan
, cookie
, txstate
);
584 if (ret
== DMA_SUCCESS
|| !txstate
)
587 spin_lock_irqsave(&echan
->vchan
.lock
, flags
);
588 vdesc
= vchan_find_desc(&echan
->vchan
, cookie
);
590 txstate
->residue
= edma_desc_size(to_edma_desc(&vdesc
->tx
));
591 } else if (echan
->edesc
&& echan
->edesc
->vdesc
.tx
.cookie
== cookie
) {
592 struct edma_desc
*edesc
= echan
->edesc
;
593 txstate
->residue
= edma_desc_size(edesc
);
595 spin_unlock_irqrestore(&echan
->vchan
.lock
, flags
);
600 static void __init
edma_chan_init(struct edma_cc
*ecc
,
601 struct dma_device
*dma
,
602 struct edma_chan
*echans
)
606 for (i
= 0; i
< EDMA_CHANS
; i
++) {
607 struct edma_chan
*echan
= &echans
[i
];
608 echan
->ch_num
= EDMA_CTLR_CHAN(ecc
->ctlr
, i
);
610 echan
->vchan
.desc_free
= edma_desc_free
;
612 vchan_init(&echan
->vchan
, dma
);
614 INIT_LIST_HEAD(&echan
->node
);
615 for (j
= 0; j
< EDMA_MAX_SLOTS
; j
++)
620 static void edma_dma_init(struct edma_cc
*ecc
, struct dma_device
*dma
,
623 dma
->device_prep_slave_sg
= edma_prep_slave_sg
;
624 dma
->device_alloc_chan_resources
= edma_alloc_chan_resources
;
625 dma
->device_free_chan_resources
= edma_free_chan_resources
;
626 dma
->device_issue_pending
= edma_issue_pending
;
627 dma
->device_tx_status
= edma_tx_status
;
628 dma
->device_control
= edma_control
;
631 INIT_LIST_HEAD(&dma
->channels
);
634 static int edma_probe(struct platform_device
*pdev
)
639 ecc
= devm_kzalloc(&pdev
->dev
, sizeof(*ecc
), GFP_KERNEL
);
641 dev_err(&pdev
->dev
, "Can't allocate controller\n");
645 ecc
->ctlr
= pdev
->id
;
646 ecc
->dummy_slot
= edma_alloc_slot(ecc
->ctlr
, EDMA_SLOT_ANY
);
647 if (ecc
->dummy_slot
< 0) {
648 dev_err(&pdev
->dev
, "Can't allocate PaRAM dummy slot\n");
652 dma_cap_zero(ecc
->dma_slave
.cap_mask
);
653 dma_cap_set(DMA_SLAVE
, ecc
->dma_slave
.cap_mask
);
655 edma_dma_init(ecc
, &ecc
->dma_slave
, &pdev
->dev
);
657 edma_chan_init(ecc
, &ecc
->dma_slave
, ecc
->slave_chans
);
659 ret
= dma_async_device_register(&ecc
->dma_slave
);
663 platform_set_drvdata(pdev
, ecc
);
665 dev_info(&pdev
->dev
, "TI EDMA DMA engine driver\n");
670 edma_free_slot(ecc
->dummy_slot
);
674 static int edma_remove(struct platform_device
*pdev
)
676 struct device
*dev
= &pdev
->dev
;
677 struct edma_cc
*ecc
= dev_get_drvdata(dev
);
679 dma_async_device_unregister(&ecc
->dma_slave
);
680 edma_free_slot(ecc
->dummy_slot
);
685 static struct platform_driver edma_driver
= {
687 .remove
= edma_remove
,
689 .name
= "edma-dma-engine",
690 .owner
= THIS_MODULE
,
694 bool edma_filter_fn(struct dma_chan
*chan
, void *param
)
696 if (chan
->device
->dev
->driver
== &edma_driver
.driver
) {
697 struct edma_chan
*echan
= to_edma_chan(chan
);
698 unsigned ch_req
= *(unsigned *)param
;
699 return ch_req
== echan
->ch_num
;
703 EXPORT_SYMBOL(edma_filter_fn
);
705 static struct platform_device
*pdev0
, *pdev1
;
707 static const struct platform_device_info edma_dev_info0
= {
708 .name
= "edma-dma-engine",
712 static const struct platform_device_info edma_dev_info1
= {
713 .name
= "edma-dma-engine",
717 static int edma_init(void)
719 int ret
= platform_driver_register(&edma_driver
);
722 pdev0
= platform_device_register_full(&edma_dev_info0
);
724 platform_driver_unregister(&edma_driver
);
725 ret
= PTR_ERR(pdev0
);
728 pdev0
->dev
.dma_mask
= &pdev0
->dev
.coherent_dma_mask
;
729 pdev0
->dev
.coherent_dma_mask
= DMA_BIT_MASK(32);
732 if (EDMA_CTLRS
== 2) {
733 pdev1
= platform_device_register_full(&edma_dev_info1
);
735 platform_driver_unregister(&edma_driver
);
736 platform_device_unregister(pdev0
);
737 ret
= PTR_ERR(pdev1
);
739 pdev1
->dev
.dma_mask
= &pdev1
->dev
.coherent_dma_mask
;
740 pdev1
->dev
.coherent_dma_mask
= DMA_BIT_MASK(32);
746 subsys_initcall(edma_init
);
748 static void __exit
edma_exit(void)
750 platform_device_unregister(pdev0
);
752 platform_device_unregister(pdev1
);
753 platform_driver_unregister(&edma_driver
);
755 module_exit(edma_exit
);
757 MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
758 MODULE_DESCRIPTION("TI EDMA DMA engine driver");
759 MODULE_LICENSE("GPL v2");