2 * MOXA ART SoCs DMA Engine support.
4 * Copyright (C) 2013 Jonas Jensen
6 * Jonas Jensen <jonas.jensen@gmail.com>
8 * This file is licensed under the terms of the GNU General Public
9 * License version 2. This program is licensed "as is" without any
10 * warranty of any kind, whether express or implied.
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/init.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/of_address.h>
24 #include <linux/of_irq.h>
25 #include <linux/of_dma.h>
26 #include <linux/bitops.h>
28 #include <asm/cacheflush.h>
30 #include "dmaengine.h"
33 #define APB_DMA_MAX_CHANNEL 4
35 #define REG_OFF_ADDRESS_SOURCE 0
36 #define REG_OFF_ADDRESS_DEST 4
37 #define REG_OFF_CYCLES 8
38 #define REG_OFF_CTRL 12
39 #define REG_OFF_CHAN_SIZE 16
41 #define APB_DMA_ENABLE BIT(0)
42 #define APB_DMA_FIN_INT_STS BIT(1)
43 #define APB_DMA_FIN_INT_EN BIT(2)
44 #define APB_DMA_BURST_MODE BIT(3)
45 #define APB_DMA_ERR_INT_STS BIT(4)
46 #define APB_DMA_ERR_INT_EN BIT(5)
52 #define APB_DMA_SOURCE_SELECT 0x40
53 #define APB_DMA_DEST_SELECT 0x80
55 #define APB_DMA_SOURCE 0x100
56 #define APB_DMA_DEST 0x1000
58 #define APB_DMA_SOURCE_MASK 0x700
59 #define APB_DMA_DEST_MASK 0x7000
63 * 001: +1 (Burst=0), +4 (Burst=1)
64 * 010: +2 (Burst=0), +8 (Burst=1)
65 * 011: +4 (Burst=0), +16 (Burst=1)
66 * 101: -1 (Burst=0), -4 (Burst=1)
67 * 110: -2 (Burst=0), -8 (Burst=1)
68 * 111: -4 (Burst=0), -16 (Burst=1)
70 #define APB_DMA_SOURCE_INC_0 0
71 #define APB_DMA_SOURCE_INC_1_4 0x100
72 #define APB_DMA_SOURCE_INC_2_8 0x200
73 #define APB_DMA_SOURCE_INC_4_16 0x300
74 #define APB_DMA_SOURCE_DEC_1_4 0x500
75 #define APB_DMA_SOURCE_DEC_2_8 0x600
76 #define APB_DMA_SOURCE_DEC_4_16 0x700
77 #define APB_DMA_DEST_INC_0 0
78 #define APB_DMA_DEST_INC_1_4 0x1000
79 #define APB_DMA_DEST_INC_2_8 0x2000
80 #define APB_DMA_DEST_INC_4_16 0x3000
81 #define APB_DMA_DEST_DEC_1_4 0x5000
82 #define APB_DMA_DEST_DEC_2_8 0x6000
83 #define APB_DMA_DEST_DEC_4_16 0x7000
86 * Request signal select source/destination address for DMA hardware handshake.
88 * The request line number is a property of the DMA controller itself,
89 * e.g. MMC must always request channels where dma_slave_config->slave_id is 5.
91 * 0: No request / Grant signal
92 * 1-15: Request / Grant signal
94 #define APB_DMA_SOURCE_REQ_NO 0x1000000
95 #define APB_DMA_SOURCE_REQ_NO_MASK 0xf000000
96 #define APB_DMA_DEST_REQ_NO 0x10000
97 #define APB_DMA_DEST_REQ_NO_MASK 0xf0000
99 #define APB_DMA_DATA_WIDTH 0x100000
100 #define APB_DMA_DATA_WIDTH_MASK 0x300000
102 * Data width of transfer:
108 #define APB_DMA_DATA_WIDTH_4 0
109 #define APB_DMA_DATA_WIDTH_2 0x100000
110 #define APB_DMA_DATA_WIDTH_1 0x200000
112 #define APB_DMA_CYCLES_MASK 0x00ffffff
114 #define MOXART_DMA_DATA_TYPE_S8 0x00
115 #define MOXART_DMA_DATA_TYPE_S16 0x01
116 #define MOXART_DMA_DATA_TYPE_S32 0x02
124 enum dma_transfer_direction dma_dir
;
127 unsigned int dma_cycles
;
128 struct virt_dma_desc vd
;
130 struct moxart_sg sg
[0];
134 struct virt_dma_chan vc
;
137 struct moxart_desc
*desc
;
139 struct dma_slave_config cfg
;
144 unsigned int line_reqno
;
148 struct moxart_dmadev
{
149 struct dma_device dma_slave
;
150 struct moxart_chan slave_chans
[APB_DMA_MAX_CHANNEL
];
153 struct moxart_filter_data
{
154 struct moxart_dmadev
*mdc
;
155 struct of_phandle_args
*dma_spec
;
158 static const unsigned int es_bytes
[] = {
159 [MOXART_DMA_DATA_TYPE_S8
] = 1,
160 [MOXART_DMA_DATA_TYPE_S16
] = 2,
161 [MOXART_DMA_DATA_TYPE_S32
] = 4,
164 static struct device
*chan2dev(struct dma_chan
*chan
)
166 return &chan
->dev
->device
;
169 static inline struct moxart_chan
*to_moxart_dma_chan(struct dma_chan
*c
)
171 return container_of(c
, struct moxart_chan
, vc
.chan
);
174 static inline struct moxart_desc
*to_moxart_dma_desc(
175 struct dma_async_tx_descriptor
*t
)
177 return container_of(t
, struct moxart_desc
, vd
.tx
);
180 static void moxart_dma_desc_free(struct virt_dma_desc
*vd
)
182 kfree(container_of(vd
, struct moxart_desc
, vd
));
185 static int moxart_terminate_all(struct dma_chan
*chan
)
187 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
192 dev_dbg(chan2dev(chan
), "%s: ch=%p\n", __func__
, ch
);
194 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
199 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
200 ctrl
&= ~(APB_DMA_ENABLE
| APB_DMA_FIN_INT_EN
| APB_DMA_ERR_INT_EN
);
201 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
203 vchan_get_all_descriptors(&ch
->vc
, &head
);
204 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
205 vchan_dma_desc_free_list(&ch
->vc
, &head
);
210 static int moxart_slave_config(struct dma_chan
*chan
,
211 struct dma_slave_config
*cfg
)
213 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
218 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
219 ctrl
|= APB_DMA_BURST_MODE
;
220 ctrl
&= ~(APB_DMA_DEST_MASK
| APB_DMA_SOURCE_MASK
);
221 ctrl
&= ~(APB_DMA_DEST_REQ_NO_MASK
| APB_DMA_SOURCE_REQ_NO_MASK
);
223 switch (ch
->cfg
.src_addr_width
) {
224 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
225 ctrl
|= APB_DMA_DATA_WIDTH_1
;
226 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
227 ctrl
|= APB_DMA_DEST_INC_1_4
;
229 ctrl
|= APB_DMA_SOURCE_INC_1_4
;
231 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
232 ctrl
|= APB_DMA_DATA_WIDTH_2
;
233 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
234 ctrl
|= APB_DMA_DEST_INC_2_8
;
236 ctrl
|= APB_DMA_SOURCE_INC_2_8
;
238 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
239 ctrl
&= ~APB_DMA_DATA_WIDTH
;
240 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
241 ctrl
|= APB_DMA_DEST_INC_4_16
;
243 ctrl
|= APB_DMA_SOURCE_INC_4_16
;
249 if (ch
->cfg
.direction
== DMA_MEM_TO_DEV
) {
250 ctrl
&= ~APB_DMA_DEST_SELECT
;
251 ctrl
|= APB_DMA_SOURCE_SELECT
;
252 ctrl
|= (ch
->line_reqno
<< 16 &
253 APB_DMA_DEST_REQ_NO_MASK
);
255 ctrl
|= APB_DMA_DEST_SELECT
;
256 ctrl
&= ~APB_DMA_SOURCE_SELECT
;
257 ctrl
|= (ch
->line_reqno
<< 24 &
258 APB_DMA_SOURCE_REQ_NO_MASK
);
261 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
266 static int moxart_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
275 case DMA_TERMINATE_ALL
:
276 moxart_terminate_all(chan
);
278 case DMA_SLAVE_CONFIG
:
279 ret
= moxart_slave_config(chan
, (struct dma_slave_config
*)arg
);
288 static struct dma_async_tx_descriptor
*moxart_prep_slave_sg(
289 struct dma_chan
*chan
, struct scatterlist
*sgl
,
290 unsigned int sg_len
, enum dma_transfer_direction dir
,
291 unsigned long tx_flags
, void *context
)
293 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
294 struct moxart_desc
*d
;
295 enum dma_slave_buswidth dev_width
;
297 struct scatterlist
*sgent
;
301 if (!is_slave_direction(dir
)) {
302 dev_err(chan2dev(chan
), "%s: invalid DMA direction\n",
307 if (dir
== DMA_DEV_TO_MEM
) {
308 dev_addr
= ch
->cfg
.src_addr
;
309 dev_width
= ch
->cfg
.src_addr_width
;
311 dev_addr
= ch
->cfg
.dst_addr
;
312 dev_width
= ch
->cfg
.dst_addr_width
;
316 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
317 es
= MOXART_DMA_DATA_TYPE_S8
;
319 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
320 es
= MOXART_DMA_DATA_TYPE_S16
;
322 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
323 es
= MOXART_DMA_DATA_TYPE_S32
;
326 dev_err(chan2dev(chan
), "%s: unsupported data width (%u)\n",
327 __func__
, dev_width
);
331 d
= kzalloc(sizeof(*d
) + sg_len
* sizeof(d
->sg
[0]), GFP_ATOMIC
);
336 d
->dev_addr
= dev_addr
;
339 for_each_sg(sgl
, sgent
, sg_len
, i
) {
340 d
->sg
[i
].addr
= sg_dma_address(sgent
);
341 d
->sg
[i
].len
= sg_dma_len(sgent
);
348 return vchan_tx_prep(&ch
->vc
, &d
->vd
, tx_flags
);
351 static struct dma_chan
*moxart_of_xlate(struct of_phandle_args
*dma_spec
,
352 struct of_dma
*ofdma
)
354 struct moxart_dmadev
*mdc
= ofdma
->of_dma_data
;
355 struct dma_chan
*chan
;
356 struct moxart_chan
*ch
;
358 chan
= dma_get_any_slave_channel(&mdc
->dma_slave
);
362 ch
= to_moxart_dma_chan(chan
);
363 ch
->line_reqno
= dma_spec
->args
[0];
368 static int moxart_alloc_chan_resources(struct dma_chan
*chan
)
370 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
372 dev_dbg(chan2dev(chan
), "%s: allocating channel #%u\n",
373 __func__
, ch
->ch_num
);
379 static void moxart_free_chan_resources(struct dma_chan
*chan
)
381 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
383 vchan_free_chan_resources(&ch
->vc
);
385 dev_dbg(chan2dev(chan
), "%s: freeing channel #%u\n",
386 __func__
, ch
->ch_num
);
390 static void moxart_dma_set_params(struct moxart_chan
*ch
, dma_addr_t src_addr
,
393 writel(src_addr
, ch
->base
+ REG_OFF_ADDRESS_SOURCE
);
394 writel(dst_addr
, ch
->base
+ REG_OFF_ADDRESS_DEST
);
397 static void moxart_set_transfer_params(struct moxart_chan
*ch
, unsigned int len
)
399 struct moxart_desc
*d
= ch
->desc
;
400 unsigned int sglen_div
= es_bytes
[d
->es
];
402 d
->dma_cycles
= len
>> sglen_div
;
405 * There are 4 cycles on 64 bytes copied, i.e. one cycle copies 16
406 * bytes ( when width is APB_DMAB_DATA_WIDTH_4 ).
408 writel(d
->dma_cycles
, ch
->base
+ REG_OFF_CYCLES
);
410 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: set %u DMA cycles (len=%u)\n",
411 __func__
, d
->dma_cycles
, len
);
414 static void moxart_start_dma(struct moxart_chan
*ch
)
418 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
419 ctrl
|= (APB_DMA_ENABLE
| APB_DMA_FIN_INT_EN
| APB_DMA_ERR_INT_EN
);
420 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
423 static void moxart_dma_start_sg(struct moxart_chan
*ch
, unsigned int idx
)
425 struct moxart_desc
*d
= ch
->desc
;
426 struct moxart_sg
*sg
= ch
->desc
->sg
+ idx
;
428 if (ch
->desc
->dma_dir
== DMA_MEM_TO_DEV
)
429 moxart_dma_set_params(ch
, sg
->addr
, d
->dev_addr
);
430 else if (ch
->desc
->dma_dir
== DMA_DEV_TO_MEM
)
431 moxart_dma_set_params(ch
, d
->dev_addr
, sg
->addr
);
433 moxart_set_transfer_params(ch
, sg
->len
);
435 moxart_start_dma(ch
);
438 static void moxart_dma_start_desc(struct dma_chan
*chan
)
440 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
441 struct virt_dma_desc
*vd
;
443 vd
= vchan_next_desc(&ch
->vc
);
452 ch
->desc
= to_moxart_dma_desc(&vd
->tx
);
455 moxart_dma_start_sg(ch
, 0);
458 static void moxart_issue_pending(struct dma_chan
*chan
)
460 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
463 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
464 if (vchan_issue_pending(&ch
->vc
) && !ch
->desc
)
465 moxart_dma_start_desc(chan
);
466 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
469 static size_t moxart_dma_desc_size(struct moxart_desc
*d
,
470 unsigned int completed_sgs
)
475 for (size
= i
= completed_sgs
; i
< d
->sglen
; i
++)
476 size
+= d
->sg
[i
].len
;
481 static size_t moxart_dma_desc_size_in_flight(struct moxart_chan
*ch
)
484 unsigned int completed_cycles
, cycles
;
486 size
= moxart_dma_desc_size(ch
->desc
, ch
->sgidx
);
487 cycles
= readl(ch
->base
+ REG_OFF_CYCLES
);
488 completed_cycles
= (ch
->desc
->dma_cycles
- cycles
);
489 size
-= completed_cycles
<< es_bytes
[ch
->desc
->es
];
491 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: size=%zu\n", __func__
, size
);
496 static enum dma_status
moxart_tx_status(struct dma_chan
*chan
,
498 struct dma_tx_state
*txstate
)
500 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
501 struct virt_dma_desc
*vd
;
502 struct moxart_desc
*d
;
507 * dma_cookie_status() assigns initial residue value.
509 ret
= dma_cookie_status(chan
, cookie
, txstate
);
511 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
512 vd
= vchan_find_desc(&ch
->vc
, cookie
);
514 d
= to_moxart_dma_desc(&vd
->tx
);
515 txstate
->residue
= moxart_dma_desc_size(d
, 0);
516 } else if (ch
->desc
&& ch
->desc
->vd
.tx
.cookie
== cookie
) {
517 txstate
->residue
= moxart_dma_desc_size_in_flight(ch
);
519 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
527 static void moxart_dma_init(struct dma_device
*dma
, struct device
*dev
)
529 dma
->device_prep_slave_sg
= moxart_prep_slave_sg
;
530 dma
->device_alloc_chan_resources
= moxart_alloc_chan_resources
;
531 dma
->device_free_chan_resources
= moxart_free_chan_resources
;
532 dma
->device_issue_pending
= moxart_issue_pending
;
533 dma
->device_tx_status
= moxart_tx_status
;
534 dma
->device_control
= moxart_control
;
537 INIT_LIST_HEAD(&dma
->channels
);
540 static irqreturn_t
moxart_dma_interrupt(int irq
, void *devid
)
542 struct moxart_dmadev
*mc
= devid
;
543 struct moxart_chan
*ch
= &mc
->slave_chans
[0];
548 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s\n", __func__
);
550 for (i
= 0; i
< APB_DMA_MAX_CHANNEL
; i
++, ch
++) {
554 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
556 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: ch=%p ch->base=%p ctrl=%x\n",
557 __func__
, ch
, ch
->base
, ctrl
);
559 if (ctrl
& APB_DMA_FIN_INT_STS
) {
560 ctrl
&= ~APB_DMA_FIN_INT_STS
;
562 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
563 if (++ch
->sgidx
< ch
->desc
->sglen
) {
564 moxart_dma_start_sg(ch
, ch
->sgidx
);
566 vchan_cookie_complete(&ch
->desc
->vd
);
567 moxart_dma_start_desc(&ch
->vc
.chan
);
569 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
573 if (ctrl
& APB_DMA_ERR_INT_STS
) {
574 ctrl
&= ~APB_DMA_ERR_INT_STS
;
578 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
584 static int moxart_probe(struct platform_device
*pdev
)
586 struct device
*dev
= &pdev
->dev
;
587 struct device_node
*node
= dev
->of_node
;
588 struct resource
*res
;
589 static void __iomem
*dma_base_addr
;
592 struct moxart_chan
*ch
;
593 struct moxart_dmadev
*mdc
;
595 mdc
= devm_kzalloc(dev
, sizeof(*mdc
), GFP_KERNEL
);
597 dev_err(dev
, "can't allocate DMA container\n");
601 irq
= irq_of_parse_and_map(node
, 0);
603 dev_err(dev
, "no IRQ resource\n");
607 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
608 dma_base_addr
= devm_ioremap_resource(dev
, res
);
609 if (IS_ERR(dma_base_addr
))
610 return PTR_ERR(dma_base_addr
);
612 dma_cap_zero(mdc
->dma_slave
.cap_mask
);
613 dma_cap_set(DMA_SLAVE
, mdc
->dma_slave
.cap_mask
);
614 dma_cap_set(DMA_PRIVATE
, mdc
->dma_slave
.cap_mask
);
616 moxart_dma_init(&mdc
->dma_slave
, dev
);
618 ch
= &mdc
->slave_chans
[0];
619 for (i
= 0; i
< APB_DMA_MAX_CHANNEL
; i
++, ch
++) {
621 ch
->base
= dma_base_addr
+ i
* REG_OFF_CHAN_SIZE
;
624 ch
->vc
.desc_free
= moxart_dma_desc_free
;
625 vchan_init(&ch
->vc
, &mdc
->dma_slave
);
627 dev_dbg(dev
, "%s: chs[%d]: ch->ch_num=%u ch->base=%p\n",
628 __func__
, i
, ch
->ch_num
, ch
->base
);
631 platform_set_drvdata(pdev
, mdc
);
633 ret
= devm_request_irq(dev
, irq
, moxart_dma_interrupt
, 0,
634 "moxart-dma-engine", mdc
);
636 dev_err(dev
, "devm_request_irq failed\n");
640 ret
= dma_async_device_register(&mdc
->dma_slave
);
642 dev_err(dev
, "dma_async_device_register failed\n");
646 ret
= of_dma_controller_register(node
, moxart_of_xlate
, mdc
);
648 dev_err(dev
, "of_dma_controller_register failed\n");
649 dma_async_device_unregister(&mdc
->dma_slave
);
653 dev_dbg(dev
, "%s: IRQ=%u\n", __func__
, irq
);
658 static int moxart_remove(struct platform_device
*pdev
)
660 struct moxart_dmadev
*m
= platform_get_drvdata(pdev
);
662 dma_async_device_unregister(&m
->dma_slave
);
664 if (pdev
->dev
.of_node
)
665 of_dma_controller_free(pdev
->dev
.of_node
);
670 static const struct of_device_id moxart_dma_match
[] = {
671 { .compatible
= "moxa,moxart-dma" },
675 static struct platform_driver moxart_driver
= {
676 .probe
= moxart_probe
,
677 .remove
= moxart_remove
,
679 .name
= "moxart-dma-engine",
680 .owner
= THIS_MODULE
,
681 .of_match_table
= moxart_dma_match
,
685 static int moxart_init(void)
687 return platform_driver_register(&moxart_driver
);
689 subsys_initcall(moxart_init
);
691 static void __exit
moxart_exit(void)
693 platform_driver_unregister(&moxart_driver
);
695 module_exit(moxart_exit
);
697 MODULE_AUTHOR("Jonas Jensen <jonas.jensen@gmail.com>");
698 MODULE_DESCRIPTION("MOXART DMA engine driver");
699 MODULE_LICENSE("GPL v2");