2 * MOXA ART SoCs DMA Engine support.
4 * Copyright (C) 2013 Jonas Jensen
6 * Jonas Jensen <jonas.jensen@gmail.com>
8 * This file is licensed under the terms of the GNU General Public
9 * License version 2. This program is licensed "as is" without any
10 * warranty of any kind, whether express or implied.
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/init.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/of_address.h>
24 #include <linux/of_irq.h>
25 #include <linux/of_dma.h>
26 #include <linux/bitops.h>
28 #include <asm/cacheflush.h>
30 #include "dmaengine.h"
33 #define APB_DMA_MAX_CHANNEL 4
35 #define REG_OFF_ADDRESS_SOURCE 0
36 #define REG_OFF_ADDRESS_DEST 4
37 #define REG_OFF_CYCLES 8
38 #define REG_OFF_CTRL 12
39 #define REG_OFF_CHAN_SIZE 16
41 #define APB_DMA_ENABLE BIT(0)
42 #define APB_DMA_FIN_INT_STS BIT(1)
43 #define APB_DMA_FIN_INT_EN BIT(2)
44 #define APB_DMA_BURST_MODE BIT(3)
45 #define APB_DMA_ERR_INT_STS BIT(4)
46 #define APB_DMA_ERR_INT_EN BIT(5)
52 #define APB_DMA_SOURCE_SELECT 0x40
53 #define APB_DMA_DEST_SELECT 0x80
55 #define APB_DMA_SOURCE 0x100
56 #define APB_DMA_DEST 0x1000
58 #define APB_DMA_SOURCE_MASK 0x700
59 #define APB_DMA_DEST_MASK 0x7000
63 * 001: +1 (Burst=0), +4 (Burst=1)
64 * 010: +2 (Burst=0), +8 (Burst=1)
65 * 011: +4 (Burst=0), +16 (Burst=1)
66 * 101: -1 (Burst=0), -4 (Burst=1)
67 * 110: -2 (Burst=0), -8 (Burst=1)
68 * 111: -4 (Burst=0), -16 (Burst=1)
70 #define APB_DMA_SOURCE_INC_0 0
71 #define APB_DMA_SOURCE_INC_1_4 0x100
72 #define APB_DMA_SOURCE_INC_2_8 0x200
73 #define APB_DMA_SOURCE_INC_4_16 0x300
74 #define APB_DMA_SOURCE_DEC_1_4 0x500
75 #define APB_DMA_SOURCE_DEC_2_8 0x600
76 #define APB_DMA_SOURCE_DEC_4_16 0x700
77 #define APB_DMA_DEST_INC_0 0
78 #define APB_DMA_DEST_INC_1_4 0x1000
79 #define APB_DMA_DEST_INC_2_8 0x2000
80 #define APB_DMA_DEST_INC_4_16 0x3000
81 #define APB_DMA_DEST_DEC_1_4 0x5000
82 #define APB_DMA_DEST_DEC_2_8 0x6000
83 #define APB_DMA_DEST_DEC_4_16 0x7000
86 * Request signal select source/destination address for DMA hardware handshake.
88 * The request line number is a property of the DMA controller itself,
89 * e.g. MMC must always request channels where dma_slave_config->slave_id is 5.
91 * 0: No request / Grant signal
92 * 1-15: Request / Grant signal
94 #define APB_DMA_SOURCE_REQ_NO 0x1000000
95 #define APB_DMA_SOURCE_REQ_NO_MASK 0xf000000
96 #define APB_DMA_DEST_REQ_NO 0x10000
97 #define APB_DMA_DEST_REQ_NO_MASK 0xf0000
99 #define APB_DMA_DATA_WIDTH 0x100000
100 #define APB_DMA_DATA_WIDTH_MASK 0x300000
102 * Data width of transfer:
108 #define APB_DMA_DATA_WIDTH_4 0
109 #define APB_DMA_DATA_WIDTH_2 0x100000
110 #define APB_DMA_DATA_WIDTH_1 0x200000
112 #define APB_DMA_CYCLES_MASK 0x00ffffff
114 #define MOXART_DMA_DATA_TYPE_S8 0x00
115 #define MOXART_DMA_DATA_TYPE_S16 0x01
116 #define MOXART_DMA_DATA_TYPE_S32 0x02
124 enum dma_transfer_direction dma_dir
;
127 unsigned int dma_cycles
;
128 struct virt_dma_desc vd
;
130 struct moxart_sg sg
[0];
134 struct virt_dma_chan vc
;
137 struct moxart_desc
*desc
;
139 struct dma_slave_config cfg
;
144 unsigned int line_reqno
;
148 struct moxart_dmadev
{
149 struct dma_device dma_slave
;
150 struct moxart_chan slave_chans
[APB_DMA_MAX_CHANNEL
];
153 struct moxart_filter_data
{
154 struct moxart_dmadev
*mdc
;
155 struct of_phandle_args
*dma_spec
;
158 static const unsigned int es_bytes
[] = {
159 [MOXART_DMA_DATA_TYPE_S8
] = 1,
160 [MOXART_DMA_DATA_TYPE_S16
] = 2,
161 [MOXART_DMA_DATA_TYPE_S32
] = 4,
164 static struct device
*chan2dev(struct dma_chan
*chan
)
166 return &chan
->dev
->device
;
169 static inline struct moxart_chan
*to_moxart_dma_chan(struct dma_chan
*c
)
171 return container_of(c
, struct moxart_chan
, vc
.chan
);
174 static inline struct moxart_desc
*to_moxart_dma_desc(
175 struct dma_async_tx_descriptor
*t
)
177 return container_of(t
, struct moxart_desc
, vd
.tx
);
180 static void moxart_dma_desc_free(struct virt_dma_desc
*vd
)
182 kfree(container_of(vd
, struct moxart_desc
, vd
));
185 static int moxart_terminate_all(struct dma_chan
*chan
)
187 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
192 dev_dbg(chan2dev(chan
), "%s: ch=%p\n", __func__
, ch
);
194 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
199 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
200 ctrl
&= ~(APB_DMA_ENABLE
| APB_DMA_FIN_INT_EN
| APB_DMA_ERR_INT_EN
);
201 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
203 vchan_get_all_descriptors(&ch
->vc
, &head
);
204 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
205 vchan_dma_desc_free_list(&ch
->vc
, &head
);
210 static int moxart_slave_config(struct dma_chan
*chan
,
211 struct dma_slave_config
*cfg
)
213 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
218 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
219 ctrl
|= APB_DMA_BURST_MODE
;
220 ctrl
&= ~(APB_DMA_DEST_MASK
| APB_DMA_SOURCE_MASK
);
221 ctrl
&= ~(APB_DMA_DEST_REQ_NO_MASK
| APB_DMA_SOURCE_REQ_NO_MASK
);
223 switch (ch
->cfg
.src_addr_width
) {
224 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
225 ctrl
|= APB_DMA_DATA_WIDTH_1
;
226 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
227 ctrl
|= APB_DMA_DEST_INC_1_4
;
229 ctrl
|= APB_DMA_SOURCE_INC_1_4
;
231 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
232 ctrl
|= APB_DMA_DATA_WIDTH_2
;
233 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
234 ctrl
|= APB_DMA_DEST_INC_2_8
;
236 ctrl
|= APB_DMA_SOURCE_INC_2_8
;
238 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
239 ctrl
&= ~APB_DMA_DATA_WIDTH
;
240 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
241 ctrl
|= APB_DMA_DEST_INC_4_16
;
243 ctrl
|= APB_DMA_SOURCE_INC_4_16
;
249 if (ch
->cfg
.direction
== DMA_MEM_TO_DEV
) {
250 ctrl
&= ~APB_DMA_DEST_SELECT
;
251 ctrl
|= APB_DMA_SOURCE_SELECT
;
252 ctrl
|= (ch
->line_reqno
<< 16 &
253 APB_DMA_DEST_REQ_NO_MASK
);
255 ctrl
|= APB_DMA_DEST_SELECT
;
256 ctrl
&= ~APB_DMA_SOURCE_SELECT
;
257 ctrl
|= (ch
->line_reqno
<< 24 &
258 APB_DMA_SOURCE_REQ_NO_MASK
);
261 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
266 static struct dma_async_tx_descriptor
*moxart_prep_slave_sg(
267 struct dma_chan
*chan
, struct scatterlist
*sgl
,
268 unsigned int sg_len
, enum dma_transfer_direction dir
,
269 unsigned long tx_flags
, void *context
)
271 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
272 struct moxart_desc
*d
;
273 enum dma_slave_buswidth dev_width
;
275 struct scatterlist
*sgent
;
279 if (!is_slave_direction(dir
)) {
280 dev_err(chan2dev(chan
), "%s: invalid DMA direction\n",
285 if (dir
== DMA_DEV_TO_MEM
) {
286 dev_addr
= ch
->cfg
.src_addr
;
287 dev_width
= ch
->cfg
.src_addr_width
;
289 dev_addr
= ch
->cfg
.dst_addr
;
290 dev_width
= ch
->cfg
.dst_addr_width
;
294 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
295 es
= MOXART_DMA_DATA_TYPE_S8
;
297 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
298 es
= MOXART_DMA_DATA_TYPE_S16
;
300 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
301 es
= MOXART_DMA_DATA_TYPE_S32
;
304 dev_err(chan2dev(chan
), "%s: unsupported data width (%u)\n",
305 __func__
, dev_width
);
309 d
= kzalloc(sizeof(*d
) + sg_len
* sizeof(d
->sg
[0]), GFP_ATOMIC
);
314 d
->dev_addr
= dev_addr
;
317 for_each_sg(sgl
, sgent
, sg_len
, i
) {
318 d
->sg
[i
].addr
= sg_dma_address(sgent
);
319 d
->sg
[i
].len
= sg_dma_len(sgent
);
326 return vchan_tx_prep(&ch
->vc
, &d
->vd
, tx_flags
);
329 static struct dma_chan
*moxart_of_xlate(struct of_phandle_args
*dma_spec
,
330 struct of_dma
*ofdma
)
332 struct moxart_dmadev
*mdc
= ofdma
->of_dma_data
;
333 struct dma_chan
*chan
;
334 struct moxart_chan
*ch
;
336 chan
= dma_get_any_slave_channel(&mdc
->dma_slave
);
340 ch
= to_moxart_dma_chan(chan
);
341 ch
->line_reqno
= dma_spec
->args
[0];
346 static int moxart_alloc_chan_resources(struct dma_chan
*chan
)
348 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
350 dev_dbg(chan2dev(chan
), "%s: allocating channel #%u\n",
351 __func__
, ch
->ch_num
);
357 static void moxart_free_chan_resources(struct dma_chan
*chan
)
359 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
361 vchan_free_chan_resources(&ch
->vc
);
363 dev_dbg(chan2dev(chan
), "%s: freeing channel #%u\n",
364 __func__
, ch
->ch_num
);
368 static void moxart_dma_set_params(struct moxart_chan
*ch
, dma_addr_t src_addr
,
371 writel(src_addr
, ch
->base
+ REG_OFF_ADDRESS_SOURCE
);
372 writel(dst_addr
, ch
->base
+ REG_OFF_ADDRESS_DEST
);
375 static void moxart_set_transfer_params(struct moxart_chan
*ch
, unsigned int len
)
377 struct moxart_desc
*d
= ch
->desc
;
378 unsigned int sglen_div
= es_bytes
[d
->es
];
380 d
->dma_cycles
= len
>> sglen_div
;
383 * There are 4 cycles on 64 bytes copied, i.e. one cycle copies 16
384 * bytes ( when width is APB_DMAB_DATA_WIDTH_4 ).
386 writel(d
->dma_cycles
, ch
->base
+ REG_OFF_CYCLES
);
388 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: set %u DMA cycles (len=%u)\n",
389 __func__
, d
->dma_cycles
, len
);
392 static void moxart_start_dma(struct moxart_chan
*ch
)
396 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
397 ctrl
|= (APB_DMA_ENABLE
| APB_DMA_FIN_INT_EN
| APB_DMA_ERR_INT_EN
);
398 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
401 static void moxart_dma_start_sg(struct moxart_chan
*ch
, unsigned int idx
)
403 struct moxart_desc
*d
= ch
->desc
;
404 struct moxart_sg
*sg
= ch
->desc
->sg
+ idx
;
406 if (ch
->desc
->dma_dir
== DMA_MEM_TO_DEV
)
407 moxart_dma_set_params(ch
, sg
->addr
, d
->dev_addr
);
408 else if (ch
->desc
->dma_dir
== DMA_DEV_TO_MEM
)
409 moxart_dma_set_params(ch
, d
->dev_addr
, sg
->addr
);
411 moxart_set_transfer_params(ch
, sg
->len
);
413 moxart_start_dma(ch
);
416 static void moxart_dma_start_desc(struct dma_chan
*chan
)
418 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
419 struct virt_dma_desc
*vd
;
421 vd
= vchan_next_desc(&ch
->vc
);
430 ch
->desc
= to_moxart_dma_desc(&vd
->tx
);
433 moxart_dma_start_sg(ch
, 0);
436 static void moxart_issue_pending(struct dma_chan
*chan
)
438 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
441 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
442 if (vchan_issue_pending(&ch
->vc
) && !ch
->desc
)
443 moxart_dma_start_desc(chan
);
444 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
447 static size_t moxart_dma_desc_size(struct moxart_desc
*d
,
448 unsigned int completed_sgs
)
453 for (size
= i
= completed_sgs
; i
< d
->sglen
; i
++)
454 size
+= d
->sg
[i
].len
;
459 static size_t moxart_dma_desc_size_in_flight(struct moxart_chan
*ch
)
462 unsigned int completed_cycles
, cycles
;
464 size
= moxart_dma_desc_size(ch
->desc
, ch
->sgidx
);
465 cycles
= readl(ch
->base
+ REG_OFF_CYCLES
);
466 completed_cycles
= (ch
->desc
->dma_cycles
- cycles
);
467 size
-= completed_cycles
<< es_bytes
[ch
->desc
->es
];
469 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: size=%zu\n", __func__
, size
);
474 static enum dma_status
moxart_tx_status(struct dma_chan
*chan
,
476 struct dma_tx_state
*txstate
)
478 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
479 struct virt_dma_desc
*vd
;
480 struct moxart_desc
*d
;
485 * dma_cookie_status() assigns initial residue value.
487 ret
= dma_cookie_status(chan
, cookie
, txstate
);
489 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
490 vd
= vchan_find_desc(&ch
->vc
, cookie
);
492 d
= to_moxart_dma_desc(&vd
->tx
);
493 txstate
->residue
= moxart_dma_desc_size(d
, 0);
494 } else if (ch
->desc
&& ch
->desc
->vd
.tx
.cookie
== cookie
) {
495 txstate
->residue
= moxart_dma_desc_size_in_flight(ch
);
497 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
505 static void moxart_dma_init(struct dma_device
*dma
, struct device
*dev
)
507 dma
->device_prep_slave_sg
= moxart_prep_slave_sg
;
508 dma
->device_alloc_chan_resources
= moxart_alloc_chan_resources
;
509 dma
->device_free_chan_resources
= moxart_free_chan_resources
;
510 dma
->device_issue_pending
= moxart_issue_pending
;
511 dma
->device_tx_status
= moxart_tx_status
;
512 dma
->device_config
= moxart_slave_config
;
513 dma
->device_terminate_all
= moxart_terminate_all
;
516 INIT_LIST_HEAD(&dma
->channels
);
519 static irqreturn_t
moxart_dma_interrupt(int irq
, void *devid
)
521 struct moxart_dmadev
*mc
= devid
;
522 struct moxart_chan
*ch
= &mc
->slave_chans
[0];
527 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s\n", __func__
);
529 for (i
= 0; i
< APB_DMA_MAX_CHANNEL
; i
++, ch
++) {
533 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
535 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: ch=%p ch->base=%p ctrl=%x\n",
536 __func__
, ch
, ch
->base
, ctrl
);
538 if (ctrl
& APB_DMA_FIN_INT_STS
) {
539 ctrl
&= ~APB_DMA_FIN_INT_STS
;
541 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
542 if (++ch
->sgidx
< ch
->desc
->sglen
) {
543 moxart_dma_start_sg(ch
, ch
->sgidx
);
545 vchan_cookie_complete(&ch
->desc
->vd
);
546 moxart_dma_start_desc(&ch
->vc
.chan
);
548 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
552 if (ctrl
& APB_DMA_ERR_INT_STS
) {
553 ctrl
&= ~APB_DMA_ERR_INT_STS
;
557 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
563 static int moxart_probe(struct platform_device
*pdev
)
565 struct device
*dev
= &pdev
->dev
;
566 struct device_node
*node
= dev
->of_node
;
567 struct resource
*res
;
568 static void __iomem
*dma_base_addr
;
571 struct moxart_chan
*ch
;
572 struct moxart_dmadev
*mdc
;
574 mdc
= devm_kzalloc(dev
, sizeof(*mdc
), GFP_KERNEL
);
576 dev_err(dev
, "can't allocate DMA container\n");
580 irq
= irq_of_parse_and_map(node
, 0);
582 dev_err(dev
, "no IRQ resource\n");
586 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
587 dma_base_addr
= devm_ioremap_resource(dev
, res
);
588 if (IS_ERR(dma_base_addr
))
589 return PTR_ERR(dma_base_addr
);
591 dma_cap_zero(mdc
->dma_slave
.cap_mask
);
592 dma_cap_set(DMA_SLAVE
, mdc
->dma_slave
.cap_mask
);
593 dma_cap_set(DMA_PRIVATE
, mdc
->dma_slave
.cap_mask
);
595 moxart_dma_init(&mdc
->dma_slave
, dev
);
597 ch
= &mdc
->slave_chans
[0];
598 for (i
= 0; i
< APB_DMA_MAX_CHANNEL
; i
++, ch
++) {
600 ch
->base
= dma_base_addr
+ i
* REG_OFF_CHAN_SIZE
;
603 ch
->vc
.desc_free
= moxart_dma_desc_free
;
604 vchan_init(&ch
->vc
, &mdc
->dma_slave
);
606 dev_dbg(dev
, "%s: chs[%d]: ch->ch_num=%u ch->base=%p\n",
607 __func__
, i
, ch
->ch_num
, ch
->base
);
610 platform_set_drvdata(pdev
, mdc
);
612 ret
= devm_request_irq(dev
, irq
, moxart_dma_interrupt
, 0,
613 "moxart-dma-engine", mdc
);
615 dev_err(dev
, "devm_request_irq failed\n");
619 ret
= dma_async_device_register(&mdc
->dma_slave
);
621 dev_err(dev
, "dma_async_device_register failed\n");
625 ret
= of_dma_controller_register(node
, moxart_of_xlate
, mdc
);
627 dev_err(dev
, "of_dma_controller_register failed\n");
628 dma_async_device_unregister(&mdc
->dma_slave
);
632 dev_dbg(dev
, "%s: IRQ=%u\n", __func__
, irq
);
637 static int moxart_remove(struct platform_device
*pdev
)
639 struct moxart_dmadev
*m
= platform_get_drvdata(pdev
);
641 dma_async_device_unregister(&m
->dma_slave
);
643 if (pdev
->dev
.of_node
)
644 of_dma_controller_free(pdev
->dev
.of_node
);
649 static const struct of_device_id moxart_dma_match
[] = {
650 { .compatible
= "moxa,moxart-dma" },
654 static struct platform_driver moxart_driver
= {
655 .probe
= moxart_probe
,
656 .remove
= moxart_remove
,
658 .name
= "moxart-dma-engine",
659 .of_match_table
= moxart_dma_match
,
663 static int moxart_init(void)
665 return platform_driver_register(&moxart_driver
);
667 subsys_initcall(moxart_init
);
669 static void __exit
moxart_exit(void)
671 platform_driver_unregister(&moxart_driver
);
673 module_exit(moxart_exit
);
675 MODULE_AUTHOR("Jonas Jensen <jonas.jensen@gmail.com>");
676 MODULE_DESCRIPTION("MOXART DMA engine driver");
677 MODULE_LICENSE("GPL v2");