2 * Driver for STM32 DMA controller
4 * Inspired by dma-jz4740.c and tegra20-apb-dma.c
6 * Copyright (C) M'boumba Cedric Madianga 2015
7 * Author: M'boumba Cedric Madianga <cedric.madianga@gmail.com>
9 * License terms: GNU General Public License (GPL), version 2
12 #include <linux/clk.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/err.h>
17 #include <linux/init.h>
18 #include <linux/jiffies.h>
19 #include <linux/list.h>
20 #include <linux/module.h>
22 #include <linux/of_device.h>
23 #include <linux/of_dma.h>
24 #include <linux/platform_device.h>
25 #include <linux/reset.h>
26 #include <linux/sched.h>
27 #include <linux/slab.h>
31 #define STM32_DMA_LISR 0x0000 /* DMA Low Int Status Reg */
32 #define STM32_DMA_HISR 0x0004 /* DMA High Int Status Reg */
33 #define STM32_DMA_LIFCR 0x0008 /* DMA Low Int Flag Clear Reg */
34 #define STM32_DMA_HIFCR 0x000c /* DMA High Int Flag Clear Reg */
35 #define STM32_DMA_TCI BIT(5) /* Transfer Complete Interrupt */
36 #define STM32_DMA_TEI BIT(3) /* Transfer Error Interrupt */
37 #define STM32_DMA_DMEI BIT(2) /* Direct Mode Error Interrupt */
38 #define STM32_DMA_FEI BIT(0) /* FIFO Error Interrupt */
40 /* DMA Stream x Configuration Register */
41 #define STM32_DMA_SCR(x) (0x0010 + 0x18 * (x)) /* x = 0..7 */
42 #define STM32_DMA_SCR_REQ(n) ((n & 0x7) << 25)
43 #define STM32_DMA_SCR_MBURST_MASK GENMASK(24, 23)
44 #define STM32_DMA_SCR_MBURST(n) ((n & 0x3) << 23)
45 #define STM32_DMA_SCR_PBURST_MASK GENMASK(22, 21)
46 #define STM32_DMA_SCR_PBURST(n) ((n & 0x3) << 21)
47 #define STM32_DMA_SCR_PL_MASK GENMASK(17, 16)
48 #define STM32_DMA_SCR_PL(n) ((n & 0x3) << 16)
49 #define STM32_DMA_SCR_MSIZE_MASK GENMASK(14, 13)
50 #define STM32_DMA_SCR_MSIZE(n) ((n & 0x3) << 13)
51 #define STM32_DMA_SCR_PSIZE_MASK GENMASK(12, 11)
52 #define STM32_DMA_SCR_PSIZE(n) ((n & 0x3) << 11)
53 #define STM32_DMA_SCR_PSIZE_GET(n) ((n & STM32_DMA_SCR_PSIZE_MASK) >> 11)
54 #define STM32_DMA_SCR_DIR_MASK GENMASK(7, 6)
55 #define STM32_DMA_SCR_DIR(n) ((n & 0x3) << 6)
56 #define STM32_DMA_SCR_CT BIT(19) /* Target in double buffer */
57 #define STM32_DMA_SCR_DBM BIT(18) /* Double Buffer Mode */
58 #define STM32_DMA_SCR_PINCOS BIT(15) /* Peripheral inc offset size */
59 #define STM32_DMA_SCR_MINC BIT(10) /* Memory increment mode */
60 #define STM32_DMA_SCR_PINC BIT(9) /* Peripheral increment mode */
61 #define STM32_DMA_SCR_CIRC BIT(8) /* Circular mode */
62 #define STM32_DMA_SCR_PFCTRL BIT(5) /* Peripheral Flow Controller */
63 #define STM32_DMA_SCR_TCIE BIT(4) /* Transfer Cplete Int Enable*/
64 #define STM32_DMA_SCR_TEIE BIT(2) /* Transfer Error Int Enable */
65 #define STM32_DMA_SCR_DMEIE BIT(1) /* Direct Mode Err Int Enable */
66 #define STM32_DMA_SCR_EN BIT(0) /* Stream Enable */
67 #define STM32_DMA_SCR_CFG_MASK (STM32_DMA_SCR_PINC \
68 | STM32_DMA_SCR_MINC \
69 | STM32_DMA_SCR_PINCOS \
70 | STM32_DMA_SCR_PL_MASK)
71 #define STM32_DMA_SCR_IRQ_MASK (STM32_DMA_SCR_TCIE \
72 | STM32_DMA_SCR_TEIE \
73 | STM32_DMA_SCR_DMEIE)
75 /* DMA Stream x number of data register */
76 #define STM32_DMA_SNDTR(x) (0x0014 + 0x18 * (x))
78 /* DMA stream peripheral address register */
79 #define STM32_DMA_SPAR(x) (0x0018 + 0x18 * (x))
81 /* DMA stream x memory 0 address register */
82 #define STM32_DMA_SM0AR(x) (0x001c + 0x18 * (x))
84 /* DMA stream x memory 1 address register */
85 #define STM32_DMA_SM1AR(x) (0x0020 + 0x18 * (x))
87 /* DMA stream x FIFO control register */
88 #define STM32_DMA_SFCR(x) (0x0024 + 0x18 * (x))
89 #define STM32_DMA_SFCR_FTH_MASK GENMASK(1, 0)
90 #define STM32_DMA_SFCR_FTH(n) (n & STM32_DMA_SFCR_FTH_MASK)
91 #define STM32_DMA_SFCR_FEIE BIT(7) /* FIFO error interrupt enable */
92 #define STM32_DMA_SFCR_DMDIS BIT(2) /* Direct mode disable */
93 #define STM32_DMA_SFCR_MASK (STM32_DMA_SFCR_FEIE \
94 | STM32_DMA_SFCR_DMDIS)
97 #define STM32_DMA_DEV_TO_MEM 0x00
98 #define STM32_DMA_MEM_TO_DEV 0x01
99 #define STM32_DMA_MEM_TO_MEM 0x02
101 /* DMA priority level */
102 #define STM32_DMA_PRIORITY_LOW 0x00
103 #define STM32_DMA_PRIORITY_MEDIUM 0x01
104 #define STM32_DMA_PRIORITY_HIGH 0x02
105 #define STM32_DMA_PRIORITY_VERY_HIGH 0x03
107 /* DMA FIFO threshold selection */
108 #define STM32_DMA_FIFO_THRESHOLD_1QUARTERFULL 0x00
109 #define STM32_DMA_FIFO_THRESHOLD_HALFFULL 0x01
110 #define STM32_DMA_FIFO_THRESHOLD_3QUARTERSFULL 0x02
111 #define STM32_DMA_FIFO_THRESHOLD_FULL 0x03
113 #define STM32_DMA_MAX_DATA_ITEMS 0xffff
114 #define STM32_DMA_MAX_CHANNELS 0x08
115 #define STM32_DMA_MAX_REQUEST_ID 0x08
116 #define STM32_DMA_MAX_DATA_PARAM 0x03
117 #define STM32_DMA_MAX_BURST 16
119 enum stm32_dma_width
{
125 enum stm32_dma_burst_size
{
126 STM32_DMA_BURST_SINGLE
,
127 STM32_DMA_BURST_INCR4
,
128 STM32_DMA_BURST_INCR8
,
129 STM32_DMA_BURST_INCR16
,
132 struct stm32_dma_cfg
{
139 struct stm32_dma_chan_reg
{
152 struct stm32_dma_sg_req
{
154 struct stm32_dma_chan_reg chan_reg
;
157 struct stm32_dma_desc
{
158 struct virt_dma_desc vdesc
;
161 struct stm32_dma_sg_req sg_req
[];
164 struct stm32_dma_chan
{
165 struct virt_dma_chan vchan
;
170 struct stm32_dma_desc
*desc
;
172 struct dma_slave_config dma_sconfig
;
173 struct stm32_dma_chan_reg chan_reg
;
176 struct stm32_dma_device
{
177 struct dma_device ddev
;
180 struct reset_control
*rst
;
182 struct stm32_dma_chan chan
[STM32_DMA_MAX_CHANNELS
];
185 static struct stm32_dma_device
*stm32_dma_get_dev(struct stm32_dma_chan
*chan
)
187 return container_of(chan
->vchan
.chan
.device
, struct stm32_dma_device
,
191 static struct stm32_dma_chan
*to_stm32_dma_chan(struct dma_chan
*c
)
193 return container_of(c
, struct stm32_dma_chan
, vchan
.chan
);
196 static struct stm32_dma_desc
*to_stm32_dma_desc(struct virt_dma_desc
*vdesc
)
198 return container_of(vdesc
, struct stm32_dma_desc
, vdesc
);
201 static struct device
*chan2dev(struct stm32_dma_chan
*chan
)
203 return &chan
->vchan
.chan
.dev
->device
;
206 static u32
stm32_dma_read(struct stm32_dma_device
*dmadev
, u32 reg
)
208 return readl_relaxed(dmadev
->base
+ reg
);
211 static void stm32_dma_write(struct stm32_dma_device
*dmadev
, u32 reg
, u32 val
)
213 writel_relaxed(val
, dmadev
->base
+ reg
);
216 static struct stm32_dma_desc
*stm32_dma_alloc_desc(u32 num_sgs
)
218 return kzalloc(sizeof(struct stm32_dma_desc
) +
219 sizeof(struct stm32_dma_sg_req
) * num_sgs
, GFP_NOWAIT
);
222 static int stm32_dma_get_width(struct stm32_dma_chan
*chan
,
223 enum dma_slave_buswidth width
)
226 case DMA_SLAVE_BUSWIDTH_1_BYTE
:
227 return STM32_DMA_BYTE
;
228 case DMA_SLAVE_BUSWIDTH_2_BYTES
:
229 return STM32_DMA_HALF_WORD
;
230 case DMA_SLAVE_BUSWIDTH_4_BYTES
:
231 return STM32_DMA_WORD
;
233 dev_err(chan2dev(chan
), "Dma bus width not supported\n");
238 static int stm32_dma_get_burst(struct stm32_dma_chan
*chan
, u32 maxburst
)
243 return STM32_DMA_BURST_SINGLE
;
245 return STM32_DMA_BURST_INCR4
;
247 return STM32_DMA_BURST_INCR8
;
249 return STM32_DMA_BURST_INCR16
;
251 dev_err(chan2dev(chan
), "Dma burst size not supported\n");
256 static void stm32_dma_set_fifo_config(struct stm32_dma_chan
*chan
,
257 u32 src_maxburst
, u32 dst_maxburst
)
259 chan
->chan_reg
.dma_sfcr
&= ~STM32_DMA_SFCR_MASK
;
260 chan
->chan_reg
.dma_scr
&= ~STM32_DMA_SCR_DMEIE
;
262 if ((!src_maxburst
) && (!dst_maxburst
)) {
263 /* Using direct mode */
264 chan
->chan_reg
.dma_scr
|= STM32_DMA_SCR_DMEIE
;
266 /* Using FIFO mode */
267 chan
->chan_reg
.dma_sfcr
|= STM32_DMA_SFCR_MASK
;
271 static int stm32_dma_slave_config(struct dma_chan
*c
,
272 struct dma_slave_config
*config
)
274 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
276 memcpy(&chan
->dma_sconfig
, config
, sizeof(*config
));
278 chan
->config_init
= true;
283 static u32
stm32_dma_irq_status(struct stm32_dma_chan
*chan
)
285 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
289 * Read "flags" from DMA_xISR register corresponding to the selected
290 * DMA channel at the correct bit offset inside that register.
292 * If (ch % 4) is 2 or 3, left shift the mask by 16 bits.
293 * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits.
297 dma_isr
= stm32_dma_read(dmadev
, STM32_DMA_HISR
);
299 dma_isr
= stm32_dma_read(dmadev
, STM32_DMA_LISR
);
301 flags
= dma_isr
>> (((chan
->id
& 2) << 3) | ((chan
->id
& 1) * 6));
306 static void stm32_dma_irq_clear(struct stm32_dma_chan
*chan
, u32 flags
)
308 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
312 * Write "flags" to the DMA_xIFCR register corresponding to the selected
313 * DMA channel at the correct bit offset inside that register.
315 * If (ch % 4) is 2 or 3, left shift the mask by 16 bits.
316 * If (ch % 4) is 1 or 3, additionally left shift the mask by 6 bits.
318 dma_ifcr
= flags
<< (((chan
->id
& 2) << 3) | ((chan
->id
& 1) * 6));
321 stm32_dma_write(dmadev
, STM32_DMA_HIFCR
, dma_ifcr
);
323 stm32_dma_write(dmadev
, STM32_DMA_LIFCR
, dma_ifcr
);
326 static int stm32_dma_disable_chan(struct stm32_dma_chan
*chan
)
328 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
329 unsigned long timeout
= jiffies
+ msecs_to_jiffies(5000);
333 dma_scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(id
));
335 if (dma_scr
& STM32_DMA_SCR_EN
) {
336 dma_scr
&= ~STM32_DMA_SCR_EN
;
337 stm32_dma_write(dmadev
, STM32_DMA_SCR(id
), dma_scr
);
340 dma_scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(id
));
341 dma_scr
&= STM32_DMA_SCR_EN
;
345 if (time_after_eq(jiffies
, timeout
)) {
346 dev_err(chan2dev(chan
), "%s: timeout!\n",
357 static void stm32_dma_stop(struct stm32_dma_chan
*chan
)
359 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
360 u32 dma_scr
, dma_sfcr
, status
;
363 /* Disable interrupts */
364 dma_scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(chan
->id
));
365 dma_scr
&= ~STM32_DMA_SCR_IRQ_MASK
;
366 stm32_dma_write(dmadev
, STM32_DMA_SCR(chan
->id
), dma_scr
);
367 dma_sfcr
= stm32_dma_read(dmadev
, STM32_DMA_SFCR(chan
->id
));
368 dma_sfcr
&= ~STM32_DMA_SFCR_FEIE
;
369 stm32_dma_write(dmadev
, STM32_DMA_SFCR(chan
->id
), dma_sfcr
);
372 ret
= stm32_dma_disable_chan(chan
);
376 /* Clear interrupt status if it is there */
377 status
= stm32_dma_irq_status(chan
);
379 dev_dbg(chan2dev(chan
), "%s(): clearing interrupt: 0x%08x\n",
381 stm32_dma_irq_clear(chan
, status
);
387 static int stm32_dma_terminate_all(struct dma_chan
*c
)
389 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
393 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
396 stm32_dma_stop(chan
);
400 vchan_get_all_descriptors(&chan
->vchan
, &head
);
401 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
402 vchan_dma_desc_free_list(&chan
->vchan
, &head
);
407 static void stm32_dma_synchronize(struct dma_chan
*c
)
409 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
411 vchan_synchronize(&chan
->vchan
);
414 static void stm32_dma_dump_reg(struct stm32_dma_chan
*chan
)
416 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
417 u32 scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(chan
->id
));
418 u32 ndtr
= stm32_dma_read(dmadev
, STM32_DMA_SNDTR(chan
->id
));
419 u32 spar
= stm32_dma_read(dmadev
, STM32_DMA_SPAR(chan
->id
));
420 u32 sm0ar
= stm32_dma_read(dmadev
, STM32_DMA_SM0AR(chan
->id
));
421 u32 sm1ar
= stm32_dma_read(dmadev
, STM32_DMA_SM1AR(chan
->id
));
422 u32 sfcr
= stm32_dma_read(dmadev
, STM32_DMA_SFCR(chan
->id
));
424 dev_dbg(chan2dev(chan
), "SCR: 0x%08x\n", scr
);
425 dev_dbg(chan2dev(chan
), "NDTR: 0x%08x\n", ndtr
);
426 dev_dbg(chan2dev(chan
), "SPAR: 0x%08x\n", spar
);
427 dev_dbg(chan2dev(chan
), "SM0AR: 0x%08x\n", sm0ar
);
428 dev_dbg(chan2dev(chan
), "SM1AR: 0x%08x\n", sm1ar
);
429 dev_dbg(chan2dev(chan
), "SFCR: 0x%08x\n", sfcr
);
432 static void stm32_dma_start_transfer(struct stm32_dma_chan
*chan
)
434 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
435 struct virt_dma_desc
*vdesc
;
436 struct stm32_dma_sg_req
*sg_req
;
437 struct stm32_dma_chan_reg
*reg
;
441 ret
= stm32_dma_disable_chan(chan
);
446 vdesc
= vchan_next_desc(&chan
->vchan
);
450 chan
->desc
= to_stm32_dma_desc(vdesc
);
454 if (chan
->next_sg
== chan
->desc
->num_sgs
)
457 sg_req
= &chan
->desc
->sg_req
[chan
->next_sg
];
458 reg
= &sg_req
->chan_reg
;
460 stm32_dma_write(dmadev
, STM32_DMA_SCR(chan
->id
), reg
->dma_scr
);
461 stm32_dma_write(dmadev
, STM32_DMA_SPAR(chan
->id
), reg
->dma_spar
);
462 stm32_dma_write(dmadev
, STM32_DMA_SM0AR(chan
->id
), reg
->dma_sm0ar
);
463 stm32_dma_write(dmadev
, STM32_DMA_SFCR(chan
->id
), reg
->dma_sfcr
);
464 stm32_dma_write(dmadev
, STM32_DMA_SM1AR(chan
->id
), reg
->dma_sm1ar
);
465 stm32_dma_write(dmadev
, STM32_DMA_SNDTR(chan
->id
), reg
->dma_sndtr
);
469 /* Clear interrupt status if it is there */
470 status
= stm32_dma_irq_status(chan
);
472 stm32_dma_irq_clear(chan
, status
);
474 stm32_dma_dump_reg(chan
);
477 reg
->dma_scr
|= STM32_DMA_SCR_EN
;
478 stm32_dma_write(dmadev
, STM32_DMA_SCR(chan
->id
), reg
->dma_scr
);
482 dev_dbg(chan2dev(chan
), "vchan %p: started\n", &chan
->vchan
);
485 static void stm32_dma_configure_next_sg(struct stm32_dma_chan
*chan
)
487 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
488 struct stm32_dma_sg_req
*sg_req
;
489 u32 dma_scr
, dma_sm0ar
, dma_sm1ar
, id
;
492 dma_scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(id
));
494 if (dma_scr
& STM32_DMA_SCR_DBM
) {
495 if (chan
->next_sg
== chan
->desc
->num_sgs
)
498 sg_req
= &chan
->desc
->sg_req
[chan
->next_sg
];
500 if (dma_scr
& STM32_DMA_SCR_CT
) {
501 dma_sm0ar
= sg_req
->chan_reg
.dma_sm0ar
;
502 stm32_dma_write(dmadev
, STM32_DMA_SM0AR(id
), dma_sm0ar
);
503 dev_dbg(chan2dev(chan
), "CT=1 <=> SM0AR: 0x%08x\n",
504 stm32_dma_read(dmadev
, STM32_DMA_SM0AR(id
)));
506 dma_sm1ar
= sg_req
->chan_reg
.dma_sm1ar
;
507 stm32_dma_write(dmadev
, STM32_DMA_SM1AR(id
), dma_sm1ar
);
508 dev_dbg(chan2dev(chan
), "CT=0 <=> SM1AR: 0x%08x\n",
509 stm32_dma_read(dmadev
, STM32_DMA_SM1AR(id
)));
514 static void stm32_dma_handle_chan_done(struct stm32_dma_chan
*chan
)
517 if (chan
->desc
->cyclic
) {
518 vchan_cyclic_callback(&chan
->desc
->vdesc
);
520 stm32_dma_configure_next_sg(chan
);
523 if (chan
->next_sg
== chan
->desc
->num_sgs
) {
524 list_del(&chan
->desc
->vdesc
.node
);
525 vchan_cookie_complete(&chan
->desc
->vdesc
);
528 stm32_dma_start_transfer(chan
);
533 static irqreturn_t
stm32_dma_chan_irq(int irq
, void *devid
)
535 struct stm32_dma_chan
*chan
= devid
;
536 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
539 spin_lock(&chan
->vchan
.lock
);
541 status
= stm32_dma_irq_status(chan
);
542 scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(chan
->id
));
544 if ((status
& STM32_DMA_TCI
) && (scr
& STM32_DMA_SCR_TCIE
)) {
545 stm32_dma_irq_clear(chan
, STM32_DMA_TCI
);
546 stm32_dma_handle_chan_done(chan
);
549 stm32_dma_irq_clear(chan
, status
);
550 dev_err(chan2dev(chan
), "DMA error: status=0x%08x\n", status
);
553 spin_unlock(&chan
->vchan
.lock
);
558 static void stm32_dma_issue_pending(struct dma_chan
*c
)
560 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
563 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
564 if (vchan_issue_pending(&chan
->vchan
) && !chan
->desc
&& !chan
->busy
) {
565 dev_dbg(chan2dev(chan
), "vchan %p: issued\n", &chan
->vchan
);
566 stm32_dma_start_transfer(chan
);
567 if (chan
->desc
->cyclic
)
568 stm32_dma_configure_next_sg(chan
);
570 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
573 static int stm32_dma_set_xfer_param(struct stm32_dma_chan
*chan
,
574 enum dma_transfer_direction direction
,
575 enum dma_slave_buswidth
*buswidth
)
577 enum dma_slave_buswidth src_addr_width
, dst_addr_width
;
578 int src_bus_width
, dst_bus_width
;
579 int src_burst_size
, dst_burst_size
;
580 u32 src_maxburst
, dst_maxburst
;
583 src_addr_width
= chan
->dma_sconfig
.src_addr_width
;
584 dst_addr_width
= chan
->dma_sconfig
.dst_addr_width
;
585 src_maxburst
= chan
->dma_sconfig
.src_maxburst
;
586 dst_maxburst
= chan
->dma_sconfig
.dst_maxburst
;
590 dst_bus_width
= stm32_dma_get_width(chan
, dst_addr_width
);
591 if (dst_bus_width
< 0)
592 return dst_bus_width
;
594 dst_burst_size
= stm32_dma_get_burst(chan
, dst_maxburst
);
595 if (dst_burst_size
< 0)
596 return dst_burst_size
;
599 src_addr_width
= dst_addr_width
;
601 src_bus_width
= stm32_dma_get_width(chan
, src_addr_width
);
602 if (src_bus_width
< 0)
603 return src_bus_width
;
605 src_burst_size
= stm32_dma_get_burst(chan
, src_maxburst
);
606 if (src_burst_size
< 0)
607 return src_burst_size
;
609 dma_scr
= STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_DEV
) |
610 STM32_DMA_SCR_PSIZE(dst_bus_width
) |
611 STM32_DMA_SCR_MSIZE(src_bus_width
) |
612 STM32_DMA_SCR_PBURST(dst_burst_size
) |
613 STM32_DMA_SCR_MBURST(src_burst_size
);
615 chan
->chan_reg
.dma_spar
= chan
->dma_sconfig
.dst_addr
;
616 *buswidth
= dst_addr_width
;
620 src_bus_width
= stm32_dma_get_width(chan
, src_addr_width
);
621 if (src_bus_width
< 0)
622 return src_bus_width
;
624 src_burst_size
= stm32_dma_get_burst(chan
, src_maxburst
);
625 if (src_burst_size
< 0)
626 return src_burst_size
;
629 dst_addr_width
= src_addr_width
;
631 dst_bus_width
= stm32_dma_get_width(chan
, dst_addr_width
);
632 if (dst_bus_width
< 0)
633 return dst_bus_width
;
635 dst_burst_size
= stm32_dma_get_burst(chan
, dst_maxburst
);
636 if (dst_burst_size
< 0)
637 return dst_burst_size
;
639 dma_scr
= STM32_DMA_SCR_DIR(STM32_DMA_DEV_TO_MEM
) |
640 STM32_DMA_SCR_PSIZE(src_bus_width
) |
641 STM32_DMA_SCR_MSIZE(dst_bus_width
) |
642 STM32_DMA_SCR_PBURST(src_burst_size
) |
643 STM32_DMA_SCR_MBURST(dst_burst_size
);
645 chan
->chan_reg
.dma_spar
= chan
->dma_sconfig
.src_addr
;
646 *buswidth
= chan
->dma_sconfig
.src_addr_width
;
650 dev_err(chan2dev(chan
), "Dma direction is not supported\n");
654 stm32_dma_set_fifo_config(chan
, src_maxburst
, dst_maxburst
);
656 chan
->chan_reg
.dma_scr
&= ~(STM32_DMA_SCR_DIR_MASK
|
657 STM32_DMA_SCR_PSIZE_MASK
| STM32_DMA_SCR_MSIZE_MASK
|
658 STM32_DMA_SCR_PBURST_MASK
| STM32_DMA_SCR_MBURST_MASK
);
659 chan
->chan_reg
.dma_scr
|= dma_scr
;
664 static void stm32_dma_clear_reg(struct stm32_dma_chan_reg
*regs
)
666 memset(regs
, 0, sizeof(struct stm32_dma_chan_reg
));
669 static struct dma_async_tx_descriptor
*stm32_dma_prep_slave_sg(
670 struct dma_chan
*c
, struct scatterlist
*sgl
,
671 u32 sg_len
, enum dma_transfer_direction direction
,
672 unsigned long flags
, void *context
)
674 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
675 struct stm32_dma_desc
*desc
;
676 struct scatterlist
*sg
;
677 enum dma_slave_buswidth buswidth
;
681 if (!chan
->config_init
) {
682 dev_err(chan2dev(chan
), "dma channel is not configured\n");
687 dev_err(chan2dev(chan
), "Invalid segment length %d\n", sg_len
);
691 desc
= stm32_dma_alloc_desc(sg_len
);
695 ret
= stm32_dma_set_xfer_param(chan
, direction
, &buswidth
);
699 /* Set peripheral flow controller */
700 if (chan
->dma_sconfig
.device_fc
)
701 chan
->chan_reg
.dma_scr
|= STM32_DMA_SCR_PFCTRL
;
703 chan
->chan_reg
.dma_scr
&= ~STM32_DMA_SCR_PFCTRL
;
705 for_each_sg(sgl
, sg
, sg_len
, i
) {
706 desc
->sg_req
[i
].len
= sg_dma_len(sg
);
708 nb_data_items
= desc
->sg_req
[i
].len
/ buswidth
;
709 if (nb_data_items
> STM32_DMA_MAX_DATA_ITEMS
) {
710 dev_err(chan2dev(chan
), "nb items not supported\n");
714 stm32_dma_clear_reg(&desc
->sg_req
[i
].chan_reg
);
715 desc
->sg_req
[i
].chan_reg
.dma_scr
= chan
->chan_reg
.dma_scr
;
716 desc
->sg_req
[i
].chan_reg
.dma_sfcr
= chan
->chan_reg
.dma_sfcr
;
717 desc
->sg_req
[i
].chan_reg
.dma_spar
= chan
->chan_reg
.dma_spar
;
718 desc
->sg_req
[i
].chan_reg
.dma_sm0ar
= sg_dma_address(sg
);
719 desc
->sg_req
[i
].chan_reg
.dma_sm1ar
= sg_dma_address(sg
);
720 desc
->sg_req
[i
].chan_reg
.dma_sndtr
= nb_data_items
;
723 desc
->num_sgs
= sg_len
;
724 desc
->cyclic
= false;
726 return vchan_tx_prep(&chan
->vchan
, &desc
->vdesc
, flags
);
733 static struct dma_async_tx_descriptor
*stm32_dma_prep_dma_cyclic(
734 struct dma_chan
*c
, dma_addr_t buf_addr
, size_t buf_len
,
735 size_t period_len
, enum dma_transfer_direction direction
,
738 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
739 struct stm32_dma_desc
*desc
;
740 enum dma_slave_buswidth buswidth
;
741 u32 num_periods
, nb_data_items
;
744 if (!buf_len
|| !period_len
) {
745 dev_err(chan2dev(chan
), "Invalid buffer/period len\n");
749 if (!chan
->config_init
) {
750 dev_err(chan2dev(chan
), "dma channel is not configured\n");
754 if (buf_len
% period_len
) {
755 dev_err(chan2dev(chan
), "buf_len not multiple of period_len\n");
760 * We allow to take more number of requests till DMA is
761 * not started. The driver will loop over all requests.
762 * Once DMA is started then new requests can be queued only after
763 * terminating the DMA.
766 dev_err(chan2dev(chan
), "Request not allowed when dma busy\n");
770 ret
= stm32_dma_set_xfer_param(chan
, direction
, &buswidth
);
774 nb_data_items
= period_len
/ buswidth
;
775 if (nb_data_items
> STM32_DMA_MAX_DATA_ITEMS
) {
776 dev_err(chan2dev(chan
), "number of items not supported\n");
780 /* Enable Circular mode or double buffer mode */
781 if (buf_len
== period_len
)
782 chan
->chan_reg
.dma_scr
|= STM32_DMA_SCR_CIRC
;
784 chan
->chan_reg
.dma_scr
|= STM32_DMA_SCR_DBM
;
786 /* Clear periph ctrl if client set it */
787 chan
->chan_reg
.dma_scr
&= ~STM32_DMA_SCR_PFCTRL
;
789 num_periods
= buf_len
/ period_len
;
791 desc
= stm32_dma_alloc_desc(num_periods
);
795 for (i
= 0; i
< num_periods
; i
++) {
796 desc
->sg_req
[i
].len
= period_len
;
798 stm32_dma_clear_reg(&desc
->sg_req
[i
].chan_reg
);
799 desc
->sg_req
[i
].chan_reg
.dma_scr
= chan
->chan_reg
.dma_scr
;
800 desc
->sg_req
[i
].chan_reg
.dma_sfcr
= chan
->chan_reg
.dma_sfcr
;
801 desc
->sg_req
[i
].chan_reg
.dma_spar
= chan
->chan_reg
.dma_spar
;
802 desc
->sg_req
[i
].chan_reg
.dma_sm0ar
= buf_addr
;
803 desc
->sg_req
[i
].chan_reg
.dma_sm1ar
= buf_addr
;
804 desc
->sg_req
[i
].chan_reg
.dma_sndtr
= nb_data_items
;
805 buf_addr
+= period_len
;
808 desc
->num_sgs
= num_periods
;
811 return vchan_tx_prep(&chan
->vchan
, &desc
->vdesc
, flags
);
814 static struct dma_async_tx_descriptor
*stm32_dma_prep_dma_memcpy(
815 struct dma_chan
*c
, dma_addr_t dest
,
816 dma_addr_t src
, size_t len
, unsigned long flags
)
818 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
820 struct stm32_dma_desc
*desc
;
821 size_t xfer_count
, offset
;
824 num_sgs
= DIV_ROUND_UP(len
, STM32_DMA_MAX_DATA_ITEMS
);
825 desc
= stm32_dma_alloc_desc(num_sgs
);
829 for (offset
= 0, i
= 0; offset
< len
; offset
+= xfer_count
, i
++) {
830 xfer_count
= min_t(size_t, len
- offset
,
831 STM32_DMA_MAX_DATA_ITEMS
);
833 desc
->sg_req
[i
].len
= xfer_count
;
835 stm32_dma_clear_reg(&desc
->sg_req
[i
].chan_reg
);
836 desc
->sg_req
[i
].chan_reg
.dma_scr
=
837 STM32_DMA_SCR_DIR(STM32_DMA_MEM_TO_MEM
) |
842 desc
->sg_req
[i
].chan_reg
.dma_sfcr
= STM32_DMA_SFCR_DMDIS
|
843 STM32_DMA_SFCR_FTH(STM32_DMA_FIFO_THRESHOLD_FULL
) |
845 desc
->sg_req
[i
].chan_reg
.dma_spar
= src
+ offset
;
846 desc
->sg_req
[i
].chan_reg
.dma_sm0ar
= dest
+ offset
;
847 desc
->sg_req
[i
].chan_reg
.dma_sndtr
= xfer_count
;
850 desc
->num_sgs
= num_sgs
;
851 desc
->cyclic
= false;
853 return vchan_tx_prep(&chan
->vchan
, &desc
->vdesc
, flags
);
856 static u32
stm32_dma_get_remaining_bytes(struct stm32_dma_chan
*chan
)
858 u32 dma_scr
, width
, ndtr
;
859 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
861 dma_scr
= stm32_dma_read(dmadev
, STM32_DMA_SCR(chan
->id
));
862 width
= STM32_DMA_SCR_PSIZE_GET(dma_scr
);
863 ndtr
= stm32_dma_read(dmadev
, STM32_DMA_SNDTR(chan
->id
));
865 return ndtr
<< width
;
868 static size_t stm32_dma_desc_residue(struct stm32_dma_chan
*chan
,
869 struct stm32_dma_desc
*desc
,
876 * In cyclic mode, for the last period, residue = remaining bytes from
879 if (chan
->desc
->cyclic
&& next_sg
== 0)
880 return stm32_dma_get_remaining_bytes(chan
);
883 * For all other periods in cyclic mode, and in sg mode,
884 * residue = remaining bytes from NDTR + remaining periods/sg to be
887 for (i
= next_sg
; i
< desc
->num_sgs
; i
++)
888 residue
+= desc
->sg_req
[i
].len
;
889 residue
+= stm32_dma_get_remaining_bytes(chan
);
894 static enum dma_status
stm32_dma_tx_status(struct dma_chan
*c
,
896 struct dma_tx_state
*state
)
898 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
899 struct virt_dma_desc
*vdesc
;
900 enum dma_status status
;
904 status
= dma_cookie_status(c
, cookie
, state
);
905 if ((status
== DMA_COMPLETE
) || (!state
))
908 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
909 vdesc
= vchan_find_desc(&chan
->vchan
, cookie
);
910 if (chan
->desc
&& cookie
== chan
->desc
->vdesc
.tx
.cookie
)
911 residue
= stm32_dma_desc_residue(chan
, chan
->desc
,
914 residue
= stm32_dma_desc_residue(chan
,
915 to_stm32_dma_desc(vdesc
), 0);
916 dma_set_residue(state
, residue
);
918 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
923 static int stm32_dma_alloc_chan_resources(struct dma_chan
*c
)
925 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
926 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
929 chan
->config_init
= false;
930 ret
= clk_prepare_enable(dmadev
->clk
);
932 dev_err(chan2dev(chan
), "clk_prepare_enable failed: %d\n", ret
);
936 ret
= stm32_dma_disable_chan(chan
);
938 clk_disable_unprepare(dmadev
->clk
);
943 static void stm32_dma_free_chan_resources(struct dma_chan
*c
)
945 struct stm32_dma_chan
*chan
= to_stm32_dma_chan(c
);
946 struct stm32_dma_device
*dmadev
= stm32_dma_get_dev(chan
);
949 dev_dbg(chan2dev(chan
), "Freeing channel %d\n", chan
->id
);
952 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
953 stm32_dma_stop(chan
);
955 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
958 clk_disable_unprepare(dmadev
->clk
);
960 vchan_free_chan_resources(to_virt_chan(c
));
963 static void stm32_dma_desc_free(struct virt_dma_desc
*vdesc
)
965 kfree(container_of(vdesc
, struct stm32_dma_desc
, vdesc
));
968 static void stm32_dma_set_config(struct stm32_dma_chan
*chan
,
969 struct stm32_dma_cfg
*cfg
)
971 stm32_dma_clear_reg(&chan
->chan_reg
);
973 chan
->chan_reg
.dma_scr
= cfg
->stream_config
& STM32_DMA_SCR_CFG_MASK
;
974 chan
->chan_reg
.dma_scr
|= STM32_DMA_SCR_REQ(cfg
->request_line
);
976 /* Enable Interrupts */
977 chan
->chan_reg
.dma_scr
|= STM32_DMA_SCR_TEIE
| STM32_DMA_SCR_TCIE
;
979 chan
->chan_reg
.dma_sfcr
= cfg
->threshold
& STM32_DMA_SFCR_FTH_MASK
;
982 static struct dma_chan
*stm32_dma_of_xlate(struct of_phandle_args
*dma_spec
,
983 struct of_dma
*ofdma
)
985 struct stm32_dma_device
*dmadev
= ofdma
->of_dma_data
;
986 struct device
*dev
= dmadev
->ddev
.dev
;
987 struct stm32_dma_cfg cfg
;
988 struct stm32_dma_chan
*chan
;
991 if (dma_spec
->args_count
< 4) {
992 dev_err(dev
, "Bad number of cells\n");
996 cfg
.channel_id
= dma_spec
->args
[0];
997 cfg
.request_line
= dma_spec
->args
[1];
998 cfg
.stream_config
= dma_spec
->args
[2];
999 cfg
.threshold
= dma_spec
->args
[3];
1001 if ((cfg
.channel_id
>= STM32_DMA_MAX_CHANNELS
) ||
1002 (cfg
.request_line
>= STM32_DMA_MAX_REQUEST_ID
)) {
1003 dev_err(dev
, "Bad channel and/or request id\n");
1007 chan
= &dmadev
->chan
[cfg
.channel_id
];
1009 c
= dma_get_slave_channel(&chan
->vchan
.chan
);
1011 dev_err(dev
, "No more channels available\n");
1015 stm32_dma_set_config(chan
, &cfg
);
1020 static const struct of_device_id stm32_dma_of_match
[] = {
1021 { .compatible
= "st,stm32-dma", },
1024 MODULE_DEVICE_TABLE(of
, stm32_dma_of_match
);
1026 static int stm32_dma_probe(struct platform_device
*pdev
)
1028 struct stm32_dma_chan
*chan
;
1029 struct stm32_dma_device
*dmadev
;
1030 struct dma_device
*dd
;
1031 const struct of_device_id
*match
;
1032 struct resource
*res
;
1035 match
= of_match_device(stm32_dma_of_match
, &pdev
->dev
);
1037 dev_err(&pdev
->dev
, "Error: No device match found\n");
1041 dmadev
= devm_kzalloc(&pdev
->dev
, sizeof(*dmadev
), GFP_KERNEL
);
1047 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1048 dmadev
->base
= devm_ioremap_resource(&pdev
->dev
, res
);
1049 if (IS_ERR(dmadev
->base
))
1050 return PTR_ERR(dmadev
->base
);
1052 dmadev
->clk
= devm_clk_get(&pdev
->dev
, NULL
);
1053 if (IS_ERR(dmadev
->clk
)) {
1054 dev_err(&pdev
->dev
, "Error: Missing controller clock\n");
1055 return PTR_ERR(dmadev
->clk
);
1058 dmadev
->mem2mem
= of_property_read_bool(pdev
->dev
.of_node
,
1061 dmadev
->rst
= devm_reset_control_get(&pdev
->dev
, NULL
);
1062 if (!IS_ERR(dmadev
->rst
)) {
1063 reset_control_assert(dmadev
->rst
);
1065 reset_control_deassert(dmadev
->rst
);
1068 dma_cap_set(DMA_SLAVE
, dd
->cap_mask
);
1069 dma_cap_set(DMA_PRIVATE
, dd
->cap_mask
);
1070 dma_cap_set(DMA_CYCLIC
, dd
->cap_mask
);
1071 dd
->device_alloc_chan_resources
= stm32_dma_alloc_chan_resources
;
1072 dd
->device_free_chan_resources
= stm32_dma_free_chan_resources
;
1073 dd
->device_tx_status
= stm32_dma_tx_status
;
1074 dd
->device_issue_pending
= stm32_dma_issue_pending
;
1075 dd
->device_prep_slave_sg
= stm32_dma_prep_slave_sg
;
1076 dd
->device_prep_dma_cyclic
= stm32_dma_prep_dma_cyclic
;
1077 dd
->device_config
= stm32_dma_slave_config
;
1078 dd
->device_terminate_all
= stm32_dma_terminate_all
;
1079 dd
->device_synchronize
= stm32_dma_synchronize
;
1080 dd
->src_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE
) |
1081 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES
) |
1082 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
);
1083 dd
->dst_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE
) |
1084 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES
) |
1085 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
);
1086 dd
->directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
1087 dd
->residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
1088 dd
->max_burst
= STM32_DMA_MAX_BURST
;
1089 dd
->dev
= &pdev
->dev
;
1090 INIT_LIST_HEAD(&dd
->channels
);
1092 if (dmadev
->mem2mem
) {
1093 dma_cap_set(DMA_MEMCPY
, dd
->cap_mask
);
1094 dd
->device_prep_dma_memcpy
= stm32_dma_prep_dma_memcpy
;
1095 dd
->directions
|= BIT(DMA_MEM_TO_MEM
);
1098 for (i
= 0; i
< STM32_DMA_MAX_CHANNELS
; i
++) {
1099 chan
= &dmadev
->chan
[i
];
1101 chan
->vchan
.desc_free
= stm32_dma_desc_free
;
1102 vchan_init(&chan
->vchan
, dd
);
1105 ret
= dma_async_device_register(dd
);
1109 for (i
= 0; i
< STM32_DMA_MAX_CHANNELS
; i
++) {
1110 chan
= &dmadev
->chan
[i
];
1111 res
= platform_get_resource(pdev
, IORESOURCE_IRQ
, i
);
1114 dev_err(&pdev
->dev
, "No irq resource for chan %d\n", i
);
1115 goto err_unregister
;
1117 chan
->irq
= res
->start
;
1118 ret
= devm_request_irq(&pdev
->dev
, chan
->irq
,
1119 stm32_dma_chan_irq
, 0,
1120 dev_name(chan2dev(chan
)), chan
);
1123 "request_irq failed with err %d channel %d\n",
1125 goto err_unregister
;
1129 ret
= of_dma_controller_register(pdev
->dev
.of_node
,
1130 stm32_dma_of_xlate
, dmadev
);
1133 "STM32 DMA DMA OF registration failed %d\n", ret
);
1134 goto err_unregister
;
1137 platform_set_drvdata(pdev
, dmadev
);
1139 dev_info(&pdev
->dev
, "STM32 DMA driver registered\n");
1144 dma_async_device_unregister(dd
);
1149 static struct platform_driver stm32_dma_driver
= {
1151 .name
= "stm32-dma",
1152 .of_match_table
= stm32_dma_of_match
,
1156 static int __init
stm32_dma_init(void)
1158 return platform_driver_probe(&stm32_dma_driver
, stm32_dma_probe
);
1160 subsys_initcall(stm32_dma_init
);