2 * MOXA ART SoCs DMA Engine support.
4 * Copyright (C) 2013 Jonas Jensen
6 * Jonas Jensen <jonas.jensen@gmail.com>
8 * This file is licensed under the terms of the GNU General Public
9 * License version 2. This program is licensed "as is" without any
10 * warranty of any kind, whether express or implied.
13 #include <linux/dmaengine.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/init.h>
17 #include <linux/interrupt.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/platform_device.h>
21 #include <linux/slab.h>
22 #include <linux/spinlock.h>
23 #include <linux/of_address.h>
24 #include <linux/of_irq.h>
25 #include <linux/of_dma.h>
26 #include <linux/bitops.h>
28 #include <asm/cacheflush.h>
30 #include "dmaengine.h"
33 #define APB_DMA_MAX_CHANNEL 4
35 #define REG_OFF_ADDRESS_SOURCE 0
36 #define REG_OFF_ADDRESS_DEST 4
37 #define REG_OFF_CYCLES 8
38 #define REG_OFF_CTRL 12
39 #define REG_OFF_CHAN_SIZE 16
41 #define APB_DMA_ENABLE BIT(0)
42 #define APB_DMA_FIN_INT_STS BIT(1)
43 #define APB_DMA_FIN_INT_EN BIT(2)
44 #define APB_DMA_BURST_MODE BIT(3)
45 #define APB_DMA_ERR_INT_STS BIT(4)
46 #define APB_DMA_ERR_INT_EN BIT(5)
52 #define APB_DMA_SOURCE_SELECT 0x40
53 #define APB_DMA_DEST_SELECT 0x80
55 #define APB_DMA_SOURCE 0x100
56 #define APB_DMA_DEST 0x1000
58 #define APB_DMA_SOURCE_MASK 0x700
59 #define APB_DMA_DEST_MASK 0x7000
63 * 001: +1 (Burst=0), +4 (Burst=1)
64 * 010: +2 (Burst=0), +8 (Burst=1)
65 * 011: +4 (Burst=0), +16 (Burst=1)
66 * 101: -1 (Burst=0), -4 (Burst=1)
67 * 110: -2 (Burst=0), -8 (Burst=1)
68 * 111: -4 (Burst=0), -16 (Burst=1)
70 #define APB_DMA_SOURCE_INC_0 0
71 #define APB_DMA_SOURCE_INC_1_4 0x100
72 #define APB_DMA_SOURCE_INC_2_8 0x200
73 #define APB_DMA_SOURCE_INC_4_16 0x300
74 #define APB_DMA_SOURCE_DEC_1_4 0x500
75 #define APB_DMA_SOURCE_DEC_2_8 0x600
76 #define APB_DMA_SOURCE_DEC_4_16 0x700
77 #define APB_DMA_DEST_INC_0 0
78 #define APB_DMA_DEST_INC_1_4 0x1000
79 #define APB_DMA_DEST_INC_2_8 0x2000
80 #define APB_DMA_DEST_INC_4_16 0x3000
81 #define APB_DMA_DEST_DEC_1_4 0x5000
82 #define APB_DMA_DEST_DEC_2_8 0x6000
83 #define APB_DMA_DEST_DEC_4_16 0x7000
86 * Request signal select source/destination address for DMA hardware handshake.
88 * The request line number is a property of the DMA controller itself,
89 * e.g. MMC must always request channels where dma_slave_config->slave_id is 5.
91 * 0: No request / Grant signal
92 * 1-15: Request / Grant signal
94 #define APB_DMA_SOURCE_REQ_NO 0x1000000
95 #define APB_DMA_SOURCE_REQ_NO_MASK 0xf000000
96 #define APB_DMA_DEST_REQ_NO 0x10000
97 #define APB_DMA_DEST_REQ_NO_MASK 0xf0000
99 #define APB_DMA_DATA_WIDTH 0x100000
100 #define APB_DMA_DATA_WIDTH_MASK 0x300000
102 * Data width of transfer:
108 #define APB_DMA_DATA_WIDTH_4 0
109 #define APB_DMA_DATA_WIDTH_2 0x100000
110 #define APB_DMA_DATA_WIDTH_1 0x200000
112 #define APB_DMA_CYCLES_MASK 0x00ffffff
114 #define MOXART_DMA_DATA_TYPE_S8 0x00
115 #define MOXART_DMA_DATA_TYPE_S16 0x01
116 #define MOXART_DMA_DATA_TYPE_S32 0x02
124 enum dma_transfer_direction dma_dir
;
127 unsigned int dma_cycles
;
128 struct virt_dma_desc vd
;
130 struct moxart_sg sg
[0];
134 struct virt_dma_chan vc
;
137 struct moxart_desc
*desc
;
139 struct dma_slave_config cfg
;
144 unsigned int line_reqno
;
148 struct moxart_dmadev
{
149 struct dma_device dma_slave
;
150 struct moxart_chan slave_chans
[APB_DMA_MAX_CHANNEL
];
153 struct moxart_filter_data
{
154 struct moxart_dmadev
*mdc
;
155 struct of_phandle_args
*dma_spec
;
158 static const unsigned int es_bytes
[] = {
159 [MOXART_DMA_DATA_TYPE_S8
] = 1,
160 [MOXART_DMA_DATA_TYPE_S16
] = 2,
161 [MOXART_DMA_DATA_TYPE_S32
] = 4,
164 static struct device
*chan2dev(struct dma_chan
*chan
)
166 return &chan
->dev
->device
;
169 static inline struct moxart_chan
*to_moxart_dma_chan(struct dma_chan
*c
)
171 return container_of(c
, struct moxart_chan
, vc
.chan
);
174 static inline struct moxart_desc
*to_moxart_dma_desc(
175 struct dma_async_tx_descriptor
*t
)
177 return container_of(t
, struct moxart_desc
, vd
.tx
);
180 static void moxart_dma_desc_free(struct virt_dma_desc
*vd
)
182 kfree(container_of(vd
, struct moxart_desc
, vd
));
185 static int moxart_terminate_all(struct dma_chan
*chan
)
187 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
192 dev_dbg(chan2dev(chan
), "%s: ch=%p\n", __func__
, ch
);
194 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
197 moxart_dma_desc_free(&ch
->desc
->vd
);
201 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
202 ctrl
&= ~(APB_DMA_ENABLE
| APB_DMA_FIN_INT_EN
| APB_DMA_ERR_INT_EN
);
203 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
205 vchan_get_all_descriptors(&ch
->vc
, &head
);
206 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
207 vchan_dma_desc_free_list(&ch
->vc
, &head
);
212 static int moxart_slave_config(struct dma_chan
*chan
,
213 struct dma_slave_config
*cfg
)
215 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
220 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
221 ctrl
|= APB_DMA_BURST_MODE
;
222 ctrl
&= ~(APB_DMA_DEST_MASK
| APB_DMA_SOURCE_MASK
);
223 ctrl
&= ~(APB_DMA_DEST_REQ_NO_MASK
| APB_DMA_SOURCE_REQ_NO_MASK
);
225 switch (ch
->cfg
.src_addr_width
) {
226 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
227 ctrl
|= APB_DMA_DATA_WIDTH_1
;
228 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
229 ctrl
|= APB_DMA_DEST_INC_1_4
;
231 ctrl
|= APB_DMA_SOURCE_INC_1_4
;
233 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
234 ctrl
|= APB_DMA_DATA_WIDTH_2
;
235 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
236 ctrl
|= APB_DMA_DEST_INC_2_8
;
238 ctrl
|= APB_DMA_SOURCE_INC_2_8
;
240 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
241 ctrl
&= ~APB_DMA_DATA_WIDTH
;
242 if (ch
->cfg
.direction
!= DMA_MEM_TO_DEV
)
243 ctrl
|= APB_DMA_DEST_INC_4_16
;
245 ctrl
|= APB_DMA_SOURCE_INC_4_16
;
251 if (ch
->cfg
.direction
== DMA_MEM_TO_DEV
) {
252 ctrl
&= ~APB_DMA_DEST_SELECT
;
253 ctrl
|= APB_DMA_SOURCE_SELECT
;
254 ctrl
|= (ch
->line_reqno
<< 16 &
255 APB_DMA_DEST_REQ_NO_MASK
);
257 ctrl
|= APB_DMA_DEST_SELECT
;
258 ctrl
&= ~APB_DMA_SOURCE_SELECT
;
259 ctrl
|= (ch
->line_reqno
<< 24 &
260 APB_DMA_SOURCE_REQ_NO_MASK
);
263 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
268 static struct dma_async_tx_descriptor
*moxart_prep_slave_sg(
269 struct dma_chan
*chan
, struct scatterlist
*sgl
,
270 unsigned int sg_len
, enum dma_transfer_direction dir
,
271 unsigned long tx_flags
, void *context
)
273 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
274 struct moxart_desc
*d
;
275 enum dma_slave_buswidth dev_width
;
277 struct scatterlist
*sgent
;
281 if (!is_slave_direction(dir
)) {
282 dev_err(chan2dev(chan
), "%s: invalid DMA direction\n",
287 if (dir
== DMA_DEV_TO_MEM
) {
288 dev_addr
= ch
->cfg
.src_addr
;
289 dev_width
= ch
->cfg
.src_addr_width
;
291 dev_addr
= ch
->cfg
.dst_addr
;
292 dev_width
= ch
->cfg
.dst_addr_width
;
296 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
297 es
= MOXART_DMA_DATA_TYPE_S8
;
299 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
300 es
= MOXART_DMA_DATA_TYPE_S16
;
302 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
303 es
= MOXART_DMA_DATA_TYPE_S32
;
306 dev_err(chan2dev(chan
), "%s: unsupported data width (%u)\n",
307 __func__
, dev_width
);
311 d
= kzalloc(sizeof(*d
) + sg_len
* sizeof(d
->sg
[0]), GFP_ATOMIC
);
316 d
->dev_addr
= dev_addr
;
319 for_each_sg(sgl
, sgent
, sg_len
, i
) {
320 d
->sg
[i
].addr
= sg_dma_address(sgent
);
321 d
->sg
[i
].len
= sg_dma_len(sgent
);
328 return vchan_tx_prep(&ch
->vc
, &d
->vd
, tx_flags
);
331 static struct dma_chan
*moxart_of_xlate(struct of_phandle_args
*dma_spec
,
332 struct of_dma
*ofdma
)
334 struct moxart_dmadev
*mdc
= ofdma
->of_dma_data
;
335 struct dma_chan
*chan
;
336 struct moxart_chan
*ch
;
338 chan
= dma_get_any_slave_channel(&mdc
->dma_slave
);
342 ch
= to_moxart_dma_chan(chan
);
343 ch
->line_reqno
= dma_spec
->args
[0];
348 static int moxart_alloc_chan_resources(struct dma_chan
*chan
)
350 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
352 dev_dbg(chan2dev(chan
), "%s: allocating channel #%u\n",
353 __func__
, ch
->ch_num
);
359 static void moxart_free_chan_resources(struct dma_chan
*chan
)
361 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
363 vchan_free_chan_resources(&ch
->vc
);
365 dev_dbg(chan2dev(chan
), "%s: freeing channel #%u\n",
366 __func__
, ch
->ch_num
);
370 static void moxart_dma_set_params(struct moxart_chan
*ch
, dma_addr_t src_addr
,
373 writel(src_addr
, ch
->base
+ REG_OFF_ADDRESS_SOURCE
);
374 writel(dst_addr
, ch
->base
+ REG_OFF_ADDRESS_DEST
);
377 static void moxart_set_transfer_params(struct moxart_chan
*ch
, unsigned int len
)
379 struct moxart_desc
*d
= ch
->desc
;
380 unsigned int sglen_div
= es_bytes
[d
->es
];
382 d
->dma_cycles
= len
>> sglen_div
;
385 * There are 4 cycles on 64 bytes copied, i.e. one cycle copies 16
386 * bytes ( when width is APB_DMAB_DATA_WIDTH_4 ).
388 writel(d
->dma_cycles
, ch
->base
+ REG_OFF_CYCLES
);
390 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: set %u DMA cycles (len=%u)\n",
391 __func__
, d
->dma_cycles
, len
);
394 static void moxart_start_dma(struct moxart_chan
*ch
)
398 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
399 ctrl
|= (APB_DMA_ENABLE
| APB_DMA_FIN_INT_EN
| APB_DMA_ERR_INT_EN
);
400 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
403 static void moxart_dma_start_sg(struct moxart_chan
*ch
, unsigned int idx
)
405 struct moxart_desc
*d
= ch
->desc
;
406 struct moxart_sg
*sg
= ch
->desc
->sg
+ idx
;
408 if (ch
->desc
->dma_dir
== DMA_MEM_TO_DEV
)
409 moxart_dma_set_params(ch
, sg
->addr
, d
->dev_addr
);
410 else if (ch
->desc
->dma_dir
== DMA_DEV_TO_MEM
)
411 moxart_dma_set_params(ch
, d
->dev_addr
, sg
->addr
);
413 moxart_set_transfer_params(ch
, sg
->len
);
415 moxart_start_dma(ch
);
418 static void moxart_dma_start_desc(struct dma_chan
*chan
)
420 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
421 struct virt_dma_desc
*vd
;
423 vd
= vchan_next_desc(&ch
->vc
);
432 ch
->desc
= to_moxart_dma_desc(&vd
->tx
);
435 moxart_dma_start_sg(ch
, 0);
438 static void moxart_issue_pending(struct dma_chan
*chan
)
440 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
443 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
444 if (vchan_issue_pending(&ch
->vc
) && !ch
->desc
)
445 moxart_dma_start_desc(chan
);
446 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
449 static size_t moxart_dma_desc_size(struct moxart_desc
*d
,
450 unsigned int completed_sgs
)
455 for (size
= i
= completed_sgs
; i
< d
->sglen
; i
++)
456 size
+= d
->sg
[i
].len
;
461 static size_t moxart_dma_desc_size_in_flight(struct moxart_chan
*ch
)
464 unsigned int completed_cycles
, cycles
;
466 size
= moxart_dma_desc_size(ch
->desc
, ch
->sgidx
);
467 cycles
= readl(ch
->base
+ REG_OFF_CYCLES
);
468 completed_cycles
= (ch
->desc
->dma_cycles
- cycles
);
469 size
-= completed_cycles
<< es_bytes
[ch
->desc
->es
];
471 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: size=%zu\n", __func__
, size
);
476 static enum dma_status
moxart_tx_status(struct dma_chan
*chan
,
478 struct dma_tx_state
*txstate
)
480 struct moxart_chan
*ch
= to_moxart_dma_chan(chan
);
481 struct virt_dma_desc
*vd
;
482 struct moxart_desc
*d
;
487 * dma_cookie_status() assigns initial residue value.
489 ret
= dma_cookie_status(chan
, cookie
, txstate
);
491 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
492 vd
= vchan_find_desc(&ch
->vc
, cookie
);
494 d
= to_moxart_dma_desc(&vd
->tx
);
495 txstate
->residue
= moxart_dma_desc_size(d
, 0);
496 } else if (ch
->desc
&& ch
->desc
->vd
.tx
.cookie
== cookie
) {
497 txstate
->residue
= moxart_dma_desc_size_in_flight(ch
);
499 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
507 static void moxart_dma_init(struct dma_device
*dma
, struct device
*dev
)
509 dma
->device_prep_slave_sg
= moxart_prep_slave_sg
;
510 dma
->device_alloc_chan_resources
= moxart_alloc_chan_resources
;
511 dma
->device_free_chan_resources
= moxart_free_chan_resources
;
512 dma
->device_issue_pending
= moxart_issue_pending
;
513 dma
->device_tx_status
= moxart_tx_status
;
514 dma
->device_config
= moxart_slave_config
;
515 dma
->device_terminate_all
= moxart_terminate_all
;
518 INIT_LIST_HEAD(&dma
->channels
);
521 static irqreturn_t
moxart_dma_interrupt(int irq
, void *devid
)
523 struct moxart_dmadev
*mc
= devid
;
524 struct moxart_chan
*ch
= &mc
->slave_chans
[0];
529 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s\n", __func__
);
531 for (i
= 0; i
< APB_DMA_MAX_CHANNEL
; i
++, ch
++) {
535 ctrl
= readl(ch
->base
+ REG_OFF_CTRL
);
537 dev_dbg(chan2dev(&ch
->vc
.chan
), "%s: ch=%p ch->base=%p ctrl=%x\n",
538 __func__
, ch
, ch
->base
, ctrl
);
540 if (ctrl
& APB_DMA_FIN_INT_STS
) {
541 ctrl
&= ~APB_DMA_FIN_INT_STS
;
543 spin_lock_irqsave(&ch
->vc
.lock
, flags
);
544 if (++ch
->sgidx
< ch
->desc
->sglen
) {
545 moxart_dma_start_sg(ch
, ch
->sgidx
);
547 vchan_cookie_complete(&ch
->desc
->vd
);
548 moxart_dma_start_desc(&ch
->vc
.chan
);
550 spin_unlock_irqrestore(&ch
->vc
.lock
, flags
);
554 if (ctrl
& APB_DMA_ERR_INT_STS
) {
555 ctrl
&= ~APB_DMA_ERR_INT_STS
;
559 writel(ctrl
, ch
->base
+ REG_OFF_CTRL
);
565 static int moxart_probe(struct platform_device
*pdev
)
567 struct device
*dev
= &pdev
->dev
;
568 struct device_node
*node
= dev
->of_node
;
569 struct resource
*res
;
570 static void __iomem
*dma_base_addr
;
573 struct moxart_chan
*ch
;
574 struct moxart_dmadev
*mdc
;
576 mdc
= devm_kzalloc(dev
, sizeof(*mdc
), GFP_KERNEL
);
578 dev_err(dev
, "can't allocate DMA container\n");
582 irq
= irq_of_parse_and_map(node
, 0);
584 dev_err(dev
, "no IRQ resource\n");
588 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
589 dma_base_addr
= devm_ioremap_resource(dev
, res
);
590 if (IS_ERR(dma_base_addr
))
591 return PTR_ERR(dma_base_addr
);
593 dma_cap_zero(mdc
->dma_slave
.cap_mask
);
594 dma_cap_set(DMA_SLAVE
, mdc
->dma_slave
.cap_mask
);
595 dma_cap_set(DMA_PRIVATE
, mdc
->dma_slave
.cap_mask
);
597 moxart_dma_init(&mdc
->dma_slave
, dev
);
599 ch
= &mdc
->slave_chans
[0];
600 for (i
= 0; i
< APB_DMA_MAX_CHANNEL
; i
++, ch
++) {
602 ch
->base
= dma_base_addr
+ i
* REG_OFF_CHAN_SIZE
;
605 ch
->vc
.desc_free
= moxart_dma_desc_free
;
606 vchan_init(&ch
->vc
, &mdc
->dma_slave
);
608 dev_dbg(dev
, "%s: chs[%d]: ch->ch_num=%u ch->base=%p\n",
609 __func__
, i
, ch
->ch_num
, ch
->base
);
612 platform_set_drvdata(pdev
, mdc
);
614 ret
= devm_request_irq(dev
, irq
, moxart_dma_interrupt
, 0,
615 "moxart-dma-engine", mdc
);
617 dev_err(dev
, "devm_request_irq failed\n");
621 ret
= dma_async_device_register(&mdc
->dma_slave
);
623 dev_err(dev
, "dma_async_device_register failed\n");
627 ret
= of_dma_controller_register(node
, moxart_of_xlate
, mdc
);
629 dev_err(dev
, "of_dma_controller_register failed\n");
630 dma_async_device_unregister(&mdc
->dma_slave
);
634 dev_dbg(dev
, "%s: IRQ=%u\n", __func__
, irq
);
639 static int moxart_remove(struct platform_device
*pdev
)
641 struct moxart_dmadev
*m
= platform_get_drvdata(pdev
);
643 dma_async_device_unregister(&m
->dma_slave
);
645 if (pdev
->dev
.of_node
)
646 of_dma_controller_free(pdev
->dev
.of_node
);
651 static const struct of_device_id moxart_dma_match
[] = {
652 { .compatible
= "moxa,moxart-dma" },
655 MODULE_DEVICE_TABLE(of
, moxart_dma_match
);
657 static struct platform_driver moxart_driver
= {
658 .probe
= moxart_probe
,
659 .remove
= moxart_remove
,
661 .name
= "moxart-dma-engine",
662 .of_match_table
= moxart_dma_match
,
666 static int moxart_init(void)
668 return platform_driver_register(&moxart_driver
);
670 subsys_initcall(moxart_init
);
672 static void __exit
moxart_exit(void)
674 platform_driver_unregister(&moxart_driver
);
676 module_exit(moxart_exit
);
678 MODULE_AUTHOR("Jonas Jensen <jonas.jensen@gmail.com>");
679 MODULE_DESCRIPTION("MOXART DMA engine driver");
680 MODULE_LICENSE("GPL v2");