?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / dma / stm32-mdma.c
blob5838311cf990044f8a0b962590e851ccedc805f1
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 *
4 * Copyright (C) STMicroelectronics SA 2017
5 * Author(s): M'boumba Cedric Madianga <cedric.madianga@gmail.com>
6 * Pierre-Yves Mordret <pierre-yves.mordret@st.com>
7 *
8 * Driver for STM32 MDMA controller
9 *
10 * Inspired by stm32-dma.c and dma-jz4780.c
11 */
12
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/dmaengine.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmapool.h>
18 #include <linux/err.h>
19 #include <linux/init.h>
20 #include <linux/iopoll.h>
21 #include <linux/jiffies.h>
22 #include <linux/list.h>
23 #include <linux/log2.h>
24 #include <linux/module.h>
25 #include <linux/of.h>
26 #include <linux/of_device.h>
27 #include <linux/of_dma.h>
28 #include <linux/platform_device.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/reset.h>
31 #include <linux/slab.h>
32
33 #include "virt-dma.h"
34
35 /* MDMA Generic getter/setter */
36 #define STM32_MDMA_SHIFT(n) (ffs(n) - 1)
37 #define STM32_MDMA_SET(n, mask) (((n) << STM32_MDMA_SHIFT(mask)) & \
38 (mask))
39 #define STM32_MDMA_GET(n, mask) (((n) & (mask)) >> \
40 STM32_MDMA_SHIFT(mask))
41
42 #define STM32_MDMA_GISR0 0x0000 /* MDMA Int Status Reg 1 */
43 #define STM32_MDMA_GISR1 0x0004 /* MDMA Int Status Reg 2 */
44
45 /* MDMA Channel x interrupt/status register */
46 #define STM32_MDMA_CISR(x) (0x40 + 0x40 * (x)) /* x = 0..62 */
47 #define STM32_MDMA_CISR_CRQA BIT(16)
48 #define STM32_MDMA_CISR_TCIF BIT(4)
49 #define STM32_MDMA_CISR_BTIF BIT(3)
50 #define STM32_MDMA_CISR_BRTIF BIT(2)
51 #define STM32_MDMA_CISR_CTCIF BIT(1)
52 #define STM32_MDMA_CISR_TEIF BIT(0)
53
54 /* MDMA Channel x interrupt flag clear register */
55 #define STM32_MDMA_CIFCR(x) (0x44 + 0x40 * (x))
56 #define STM32_MDMA_CIFCR_CLTCIF BIT(4)
57 #define STM32_MDMA_CIFCR_CBTIF BIT(3)
58 #define STM32_MDMA_CIFCR_CBRTIF BIT(2)
59 #define STM32_MDMA_CIFCR_CCTCIF BIT(1)
60 #define STM32_MDMA_CIFCR_CTEIF BIT(0)
61 #define STM32_MDMA_CIFCR_CLEAR_ALL (STM32_MDMA_CIFCR_CLTCIF \
62 | STM32_MDMA_CIFCR_CBTIF \
63 | STM32_MDMA_CIFCR_CBRTIF \
64 | STM32_MDMA_CIFCR_CCTCIF \
65 | STM32_MDMA_CIFCR_CTEIF)
66
67 /* MDMA Channel x error status register */
68 #define STM32_MDMA_CESR(x) (0x48 + 0x40 * (x))
69 #define STM32_MDMA_CESR_BSE BIT(11)
70 #define STM32_MDMA_CESR_ASR BIT(10)
71 #define STM32_MDMA_CESR_TEMD BIT(9)
72 #define STM32_MDMA_CESR_TELD BIT(8)
73 #define STM32_MDMA_CESR_TED BIT(7)
74 #define STM32_MDMA_CESR_TEA_MASK GENMASK(6, 0)
75
76 /* MDMA Channel x control register */
77 #define STM32_MDMA_CCR(x) (0x4C + 0x40 * (x))
78 #define STM32_MDMA_CCR_SWRQ BIT(16)
79 #define STM32_MDMA_CCR_WEX BIT(14)
80 #define STM32_MDMA_CCR_HEX BIT(13)
81 #define STM32_MDMA_CCR_BEX BIT(12)
82 #define STM32_MDMA_CCR_PL_MASK GENMASK(7, 6)
83 #define STM32_MDMA_CCR_PL(n) STM32_MDMA_SET(n, \
84 STM32_MDMA_CCR_PL_MASK)
85 #define STM32_MDMA_CCR_TCIE BIT(5)
86 #define STM32_MDMA_CCR_BTIE BIT(4)
87 #define STM32_MDMA_CCR_BRTIE BIT(3)
88 #define STM32_MDMA_CCR_CTCIE BIT(2)
89 #define STM32_MDMA_CCR_TEIE BIT(1)
90 #define STM32_MDMA_CCR_EN BIT(0)
91 #define STM32_MDMA_CCR_IRQ_MASK (STM32_MDMA_CCR_TCIE \
92 | STM32_MDMA_CCR_BTIE \
93 | STM32_MDMA_CCR_BRTIE \
94 | STM32_MDMA_CCR_CTCIE \
95 | STM32_MDMA_CCR_TEIE)
96
97 /* MDMA Channel x transfer configuration register */
98 #define STM32_MDMA_CTCR(x) (0x50 + 0x40 * (x))
99 #define STM32_MDMA_CTCR_BWM BIT(31)
100 #define STM32_MDMA_CTCR_SWRM BIT(30)
101 #define STM32_MDMA_CTCR_TRGM_MSK GENMASK(29, 28)
102 #define STM32_MDMA_CTCR_TRGM(n) STM32_MDMA_SET((n), \
103 STM32_MDMA_CTCR_TRGM_MSK)
104 #define STM32_MDMA_CTCR_TRGM_GET(n) STM32_MDMA_GET((n), \
105 STM32_MDMA_CTCR_TRGM_MSK)
106 #define STM32_MDMA_CTCR_PAM_MASK GENMASK(27, 26)
107 #define STM32_MDMA_CTCR_PAM(n) STM32_MDMA_SET(n, \
108 STM32_MDMA_CTCR_PAM_MASK)
109 #define STM32_MDMA_CTCR_PKE BIT(25)
110 #define STM32_MDMA_CTCR_TLEN_MSK GENMASK(24, 18)
111 #define STM32_MDMA_CTCR_TLEN(n) STM32_MDMA_SET((n), \
112 STM32_MDMA_CTCR_TLEN_MSK)
113 #define STM32_MDMA_CTCR_TLEN_GET(n) STM32_MDMA_GET((n), \
114 STM32_MDMA_CTCR_TLEN_MSK)
115 #define STM32_MDMA_CTCR_LEN2_MSK GENMASK(25, 18)
116 #define STM32_MDMA_CTCR_LEN2(n) STM32_MDMA_SET((n), \
117 STM32_MDMA_CTCR_LEN2_MSK)
118 #define STM32_MDMA_CTCR_LEN2_GET(n) STM32_MDMA_GET((n), \
119 STM32_MDMA_CTCR_LEN2_MSK)
120 #define STM32_MDMA_CTCR_DBURST_MASK GENMASK(17, 15)
121 #define STM32_MDMA_CTCR_DBURST(n) STM32_MDMA_SET(n, \
122 STM32_MDMA_CTCR_DBURST_MASK)
123 #define STM32_MDMA_CTCR_SBURST_MASK GENMASK(14, 12)
124 #define STM32_MDMA_CTCR_SBURST(n) STM32_MDMA_SET(n, \
125 STM32_MDMA_CTCR_SBURST_MASK)
126 #define STM32_MDMA_CTCR_DINCOS_MASK GENMASK(11, 10)
127 #define STM32_MDMA_CTCR_DINCOS(n) STM32_MDMA_SET((n), \
128 STM32_MDMA_CTCR_DINCOS_MASK)
129 #define STM32_MDMA_CTCR_SINCOS_MASK GENMASK(9, 8)
130 #define STM32_MDMA_CTCR_SINCOS(n) STM32_MDMA_SET((n), \
131 STM32_MDMA_CTCR_SINCOS_MASK)
132 #define STM32_MDMA_CTCR_DSIZE_MASK GENMASK(7, 6)
133 #define STM32_MDMA_CTCR_DSIZE(n) STM32_MDMA_SET(n, \
134 STM32_MDMA_CTCR_DSIZE_MASK)
135 #define STM32_MDMA_CTCR_SSIZE_MASK GENMASK(5, 4)
136 #define STM32_MDMA_CTCR_SSIZE(n) STM32_MDMA_SET(n, \
137 STM32_MDMA_CTCR_SSIZE_MASK)
138 #define STM32_MDMA_CTCR_DINC_MASK GENMASK(3, 2)
139 #define STM32_MDMA_CTCR_DINC(n) STM32_MDMA_SET((n), \
140 STM32_MDMA_CTCR_DINC_MASK)
141 #define STM32_MDMA_CTCR_SINC_MASK GENMASK(1, 0)
142 #define STM32_MDMA_CTCR_SINC(n) STM32_MDMA_SET((n), \
143 STM32_MDMA_CTCR_SINC_MASK)
144 #define STM32_MDMA_CTCR_CFG_MASK (STM32_MDMA_CTCR_SINC_MASK \
145 | STM32_MDMA_CTCR_DINC_MASK \
146 | STM32_MDMA_CTCR_SINCOS_MASK \
147 | STM32_MDMA_CTCR_DINCOS_MASK \
148 | STM32_MDMA_CTCR_LEN2_MSK \
149 | STM32_MDMA_CTCR_TRGM_MSK)
150
151 /* MDMA Channel x block number of data register */
152 #define STM32_MDMA_CBNDTR(x) (0x54 + 0x40 * (x))
153 #define STM32_MDMA_CBNDTR_BRC_MK GENMASK(31, 20)
154 #define STM32_MDMA_CBNDTR_BRC(n) STM32_MDMA_SET(n, \
155 STM32_MDMA_CBNDTR_BRC_MK)
156 #define STM32_MDMA_CBNDTR_BRC_GET(n) STM32_MDMA_GET((n), \
157 STM32_MDMA_CBNDTR_BRC_MK)
158
159 #define STM32_MDMA_CBNDTR_BRDUM BIT(19)
160 #define STM32_MDMA_CBNDTR_BRSUM BIT(18)
161 #define STM32_MDMA_CBNDTR_BNDT_MASK GENMASK(16, 0)
162 #define STM32_MDMA_CBNDTR_BNDT(n) STM32_MDMA_SET(n, \
163 STM32_MDMA_CBNDTR_BNDT_MASK)
164
165 /* MDMA Channel x source address register */
166 #define STM32_MDMA_CSAR(x) (0x58 + 0x40 * (x))
167
168 /* MDMA Channel x destination address register */
169 #define STM32_MDMA_CDAR(x) (0x5C + 0x40 * (x))
170
171 /* MDMA Channel x block repeat address update register */
172 #define STM32_MDMA_CBRUR(x) (0x60 + 0x40 * (x))
173 #define STM32_MDMA_CBRUR_DUV_MASK GENMASK(31, 16)
174 #define STM32_MDMA_CBRUR_DUV(n) STM32_MDMA_SET(n, \
175 STM32_MDMA_CBRUR_DUV_MASK)
176 #define STM32_MDMA_CBRUR_SUV_MASK GENMASK(15, 0)
177 #define STM32_MDMA_CBRUR_SUV(n) STM32_MDMA_SET(n, \
178 STM32_MDMA_CBRUR_SUV_MASK)
179
180 /* MDMA Channel x link address register */
181 #define STM32_MDMA_CLAR(x) (0x64 + 0x40 * (x))
182
183 /* MDMA Channel x trigger and bus selection register */
184 #define STM32_MDMA_CTBR(x) (0x68 + 0x40 * (x))
185 #define STM32_MDMA_CTBR_DBUS BIT(17)
186 #define STM32_MDMA_CTBR_SBUS BIT(16)
187 #define STM32_MDMA_CTBR_TSEL_MASK GENMASK(7, 0)
188 #define STM32_MDMA_CTBR_TSEL(n) STM32_MDMA_SET(n, \
189 STM32_MDMA_CTBR_TSEL_MASK)
190
191 /* MDMA Channel x mask address register */
192 #define STM32_MDMA_CMAR(x) (0x70 + 0x40 * (x))
193
194 /* MDMA Channel x mask data register */
195 #define STM32_MDMA_CMDR(x) (0x74 + 0x40 * (x))
196
197 #define STM32_MDMA_MAX_BUF_LEN 128
198 #define STM32_MDMA_MAX_BLOCK_LEN 65536
199 #define STM32_MDMA_MAX_CHANNELS 63
200 #define STM32_MDMA_MAX_REQUESTS 256
201 #define STM32_MDMA_MAX_BURST 128
202 #define STM32_MDMA_VERY_HIGH_PRIORITY 0x11
203
204 enum stm32_mdma_trigger_mode {
205 STM32_MDMA_BUFFER,
206 STM32_MDMA_BLOCK,
207 STM32_MDMA_BLOCK_REP,
208 STM32_MDMA_LINKED_LIST,
209 };
210
211 enum stm32_mdma_width {
212 STM32_MDMA_BYTE,
213 STM32_MDMA_HALF_WORD,
214 STM32_MDMA_WORD,
215 STM32_MDMA_DOUBLE_WORD,
216 };
217
218 enum stm32_mdma_inc_mode {
219 STM32_MDMA_FIXED = 0,
220 STM32_MDMA_INC = 2,
221 STM32_MDMA_DEC = 3,
222 };
223
224 struct stm32_mdma_chan_config {
225 u32 request;
226 u32 priority_level;
227 u32 transfer_config;
228 u32 mask_addr;
229 u32 mask_data;
230 };
231
232 struct stm32_mdma_hwdesc {
233 u32 ctcr;
234 u32 cbndtr;
235 u32 csar;
236 u32 cdar;
237 u32 cbrur;
238 u32 clar;
239 u32 ctbr;
240 u32 dummy;
241 u32 cmar;
242 u32 cmdr;
243 } __aligned(64);
244
245 struct stm32_mdma_desc_node {
246 struct stm32_mdma_hwdesc *hwdesc;
247 dma_addr_t hwdesc_phys;
248 };
249
250 struct stm32_mdma_desc {
251 struct virt_dma_desc vdesc;
252 u32 ccr;
253 bool cyclic;
254 u32 count;
255 struct stm32_mdma_desc_node node[];
256 };
257
258 struct stm32_mdma_chan {
259 struct virt_dma_chan vchan;
260 struct dma_pool *desc_pool;
261 u32 id;
262 struct stm32_mdma_desc *desc;
263 u32 curr_hwdesc;
264 struct dma_slave_config dma_config;
265 struct stm32_mdma_chan_config chan_config;
266 bool busy;
267 u32 mem_burst;
268 u32 mem_width;
269 };
270
271 struct stm32_mdma_device {
272 struct dma_device ddev;
273 void __iomem *base;
274 struct clk *clk;
275 int irq;
276 struct reset_control *rst;
277 u32 nr_channels;
278 u32 nr_requests;
279 u32 nr_ahb_addr_masks;
280 struct stm32_mdma_chan chan[STM32_MDMA_MAX_CHANNELS];
281 u32 ahb_addr_masks[];
282 };
283
284 static struct stm32_mdma_device *stm32_mdma_get_dev(
285 struct stm32_mdma_chan *chan)
286 {
287 return container_of(chan->vchan.chan.device, struct stm32_mdma_device,
288 ddev);
289 }
290
291 static struct stm32_mdma_chan *to_stm32_mdma_chan(struct dma_chan *c)
292 {
293 return container_of(c, struct stm32_mdma_chan, vchan.chan);
294 }
295
296 static struct stm32_mdma_desc *to_stm32_mdma_desc(struct virt_dma_desc *vdesc)
297 {
298 return container_of(vdesc, struct stm32_mdma_desc, vdesc);
299 }
300
301 static struct device *chan2dev(struct stm32_mdma_chan *chan)
302 {
303 return &chan->vchan.chan.dev->device;
304 }
305
306 static struct device *mdma2dev(struct stm32_mdma_device *mdma_dev)
307 {
308 return mdma_dev->ddev.dev;
309 }
310
311 static u32 stm32_mdma_read(struct stm32_mdma_device *dmadev, u32 reg)
312 {
313 return readl_relaxed(dmadev->base + reg);
314 }
315
316 static void stm32_mdma_write(struct stm32_mdma_device *dmadev, u32 reg, u32 val)
317 {
318 writel_relaxed(val, dmadev->base + reg);
319 }
320
321 static void stm32_mdma_set_bits(struct stm32_mdma_device *dmadev, u32 reg,
322 u32 mask)
323 {
324 void __iomem *addr = dmadev->base + reg;
325
326 writel_relaxed(readl_relaxed(addr) | mask, addr);
327 }
328
329 static void stm32_mdma_clr_bits(struct stm32_mdma_device *dmadev, u32 reg,
330 u32 mask)
331 {
332 void __iomem *addr = dmadev->base + reg;
333
334 writel_relaxed(readl_relaxed(addr) & ~mask, addr);
335 }
336
337 static struct stm32_mdma_desc *stm32_mdma_alloc_desc(
338 struct stm32_mdma_chan *chan, u32 count)
339 {
340 struct stm32_mdma_desc *desc;
341 int i;
342
343 desc = kzalloc(offsetof(typeof(*desc), node[count]), GFP_NOWAIT);
344 if (!desc)
345 return NULL;
346
347 for (i = 0; i < count; i++) {
348 desc->node[i].hwdesc =
349 dma_pool_alloc(chan->desc_pool, GFP_NOWAIT,
350 &desc->node[i].hwdesc_phys);
351 if (!desc->node[i].hwdesc)
352 goto err;
353 }
354
355 desc->count = count;
356
357 return desc;
358
359 err:
360 dev_err(chan2dev(chan), "Failed to allocate descriptor\n");
361 while (--i >= 0)
362 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
363 desc->node[i].hwdesc_phys);
364 kfree(desc);
365 return NULL;
366 }
367
368 static void stm32_mdma_desc_free(struct virt_dma_desc *vdesc)
369 {
370 struct stm32_mdma_desc *desc = to_stm32_mdma_desc(vdesc);
371 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(vdesc->tx.chan);
372 int i;
373
374 for (i = 0; i < desc->count; i++)
375 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
376 desc->node[i].hwdesc_phys);
377 kfree(desc);
378 }
379
380 static int stm32_mdma_get_width(struct stm32_mdma_chan *chan,
381 enum dma_slave_buswidth width)
382 {
383 switch (width) {
384 case DMA_SLAVE_BUSWIDTH_1_BYTE:
385 case DMA_SLAVE_BUSWIDTH_2_BYTES:
386 case DMA_SLAVE_BUSWIDTH_4_BYTES:
387 case DMA_SLAVE_BUSWIDTH_8_BYTES:
388 return ffs(width) - 1;
389 default:
390 dev_err(chan2dev(chan), "Dma bus width %i not supported\n",
391 width);
392 return -EINVAL;
393 }
394 }
395
396 static enum dma_slave_buswidth stm32_mdma_get_max_width(dma_addr_t addr,
397 u32 buf_len, u32 tlen)
398 {
399 enum dma_slave_buswidth max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
400
401 for (max_width = DMA_SLAVE_BUSWIDTH_8_BYTES;
402 max_width > DMA_SLAVE_BUSWIDTH_1_BYTE;
403 max_width >>= 1) {
404 /*
405 * Address and buffer length both have to be aligned on
406 * bus width
407 */
408 if ((((buf_len | addr) & (max_width - 1)) == 0) &&
409 tlen >= max_width)
410 break;
411 }
412
413 return max_width;
414 }
415
416 static u32 stm32_mdma_get_best_burst(u32 buf_len, u32 tlen, u32 max_burst,
417 enum dma_slave_buswidth width)
418 {
419 u32 best_burst;
420
421 best_burst = min((u32)1 << __ffs(tlen | buf_len),
422 max_burst * width) / width;
423
424 return (best_burst > 0) ? best_burst : 1;
425 }
426
427 static int stm32_mdma_disable_chan(struct stm32_mdma_chan *chan)
428 {
429 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
430 u32 ccr, cisr, id, reg;
431 int ret;
432
433 id = chan->id;
434 reg = STM32_MDMA_CCR(id);
435
436 /* Disable interrupts */
437 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_IRQ_MASK);
438
439 ccr = stm32_mdma_read(dmadev, reg);
440 if (ccr & STM32_MDMA_CCR_EN) {
441 stm32_mdma_clr_bits(dmadev, reg, STM32_MDMA_CCR_EN);
442
443 /* Ensure that any ongoing transfer has been completed */
444 ret = readl_relaxed_poll_timeout_atomic(
445 dmadev->base + STM32_MDMA_CISR(id), cisr,
446 (cisr & STM32_MDMA_CISR_CTCIF), 10, 1000);
447 if (ret) {
448 dev_err(chan2dev(chan), "%s: timeout!\n", __func__);
449 return -EBUSY;
450 }
451 }
452
453 return 0;
454 }
455
456 static void stm32_mdma_stop(struct stm32_mdma_chan *chan)
457 {
458 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
459 u32 status;
460 int ret;
461
462 /* Disable DMA */
463 ret = stm32_mdma_disable_chan(chan);
464 if (ret < 0)
465 return;
466
467 /* Clear interrupt status if it is there */
468 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
469 if (status) {
470 dev_dbg(chan2dev(chan), "%s(): clearing interrupt: 0x%08x\n",
471 __func__, status);
472 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
473 }
474
475 chan->busy = false;
476 }
477
478 static void stm32_mdma_set_bus(struct stm32_mdma_device *dmadev, u32 *ctbr,
479 u32 ctbr_mask, u32 src_addr)
480 {
481 u32 mask;
482 int i;
483
484 /* Check if memory device is on AHB or AXI */
485 *ctbr &= ~ctbr_mask;
486 mask = src_addr & 0xF0000000;
487 for (i = 0; i < dmadev->nr_ahb_addr_masks; i++) {
488 if (mask == dmadev->ahb_addr_masks[i]) {
489 *ctbr |= ctbr_mask;
490 break;
491 }
492 }
493 }
494
495 static int stm32_mdma_set_xfer_param(struct stm32_mdma_chan *chan,
496 enum dma_transfer_direction direction,
497 u32 *mdma_ccr, u32 *mdma_ctcr,
498 u32 *mdma_ctbr, dma_addr_t addr,
499 u32 buf_len)
500 {
501 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
502 struct stm32_mdma_chan_config *chan_config = &chan->chan_config;
503 enum dma_slave_buswidth src_addr_width, dst_addr_width;
504 phys_addr_t src_addr, dst_addr;
505 int src_bus_width, dst_bus_width;
506 u32 src_maxburst, dst_maxburst, src_best_burst, dst_best_burst;
507 u32 ccr, ctcr, ctbr, tlen;
508
509 src_addr_width = chan->dma_config.src_addr_width;
510 dst_addr_width = chan->dma_config.dst_addr_width;
511 src_maxburst = chan->dma_config.src_maxburst;
512 dst_maxburst = chan->dma_config.dst_maxburst;
513
514 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
515 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
516 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
517
518 /* Enable HW request mode */
519 ctcr &= ~STM32_MDMA_CTCR_SWRM;
520
521 /* Set DINC, SINC, DINCOS, SINCOS, TRGM and TLEN retrieve from DT */
522 ctcr &= ~STM32_MDMA_CTCR_CFG_MASK;
523 ctcr |= chan_config->transfer_config & STM32_MDMA_CTCR_CFG_MASK;
524
525 /*
526 * For buffer transfer length (TLEN) we have to set
527 * the number of bytes - 1 in CTCR register
528 */
529 tlen = STM32_MDMA_CTCR_LEN2_GET(ctcr);
530 ctcr &= ~STM32_MDMA_CTCR_LEN2_MSK;
531 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
532
533 /* Disable Pack Enable */
534 ctcr &= ~STM32_MDMA_CTCR_PKE;
535
536 /* Check burst size constraints */
537 if (src_maxburst * src_addr_width > STM32_MDMA_MAX_BURST ||
538 dst_maxburst * dst_addr_width > STM32_MDMA_MAX_BURST) {
539 dev_err(chan2dev(chan),
540 "burst size * bus width higher than %d bytes\n",
541 STM32_MDMA_MAX_BURST);
542 return -EINVAL;
543 }
544
545 if ((!is_power_of_2(src_maxburst) && src_maxburst > 0) ||
546 (!is_power_of_2(dst_maxburst) && dst_maxburst > 0)) {
547 dev_err(chan2dev(chan), "burst size must be a power of 2\n");
548 return -EINVAL;
549 }
550
551 /*
552 * Configure channel control:
553 * - Clear SW request as in this case this is a HW one
554 * - Clear WEX, HEX and BEX bits
555 * - Set priority level
556 */
557 ccr &= ~(STM32_MDMA_CCR_SWRQ | STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
558 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK);
559 ccr |= STM32_MDMA_CCR_PL(chan_config->priority_level);
560
561 /* Configure Trigger selection */
562 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
563 ctbr |= STM32_MDMA_CTBR_TSEL(chan_config->request);
564
565 switch (direction) {
566 case DMA_MEM_TO_DEV:
567 dst_addr = chan->dma_config.dst_addr;
568
569 /* Set device data size */
570 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
571 if (dst_bus_width < 0)
572 return dst_bus_width;
573 ctcr &= ~STM32_MDMA_CTCR_DSIZE_MASK;
574 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width);
575
576 /* Set device burst value */
577 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
578 dst_maxburst,
579 dst_addr_width);
580 chan->mem_burst = dst_best_burst;
581 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
582 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
583
584 /* Set memory data size */
585 src_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
586 chan->mem_width = src_addr_width;
587 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
588 if (src_bus_width < 0)
589 return src_bus_width;
590 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK |
591 STM32_MDMA_CTCR_SINCOS_MASK;
592 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width) |
593 STM32_MDMA_CTCR_SINCOS(src_bus_width);
594
595 /* Set memory burst value */
596 src_maxburst = STM32_MDMA_MAX_BUF_LEN / src_addr_width;
597 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
598 src_maxburst,
599 src_addr_width);
600 chan->mem_burst = src_best_burst;
601 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
602 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
603
604 /* Select bus */
605 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
606 dst_addr);
607
608 if (dst_bus_width != src_bus_width)
609 ctcr |= STM32_MDMA_CTCR_PKE;
610
611 /* Set destination address */
612 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(chan->id), dst_addr);
613 break;
614
615 case DMA_DEV_TO_MEM:
616 src_addr = chan->dma_config.src_addr;
617
618 /* Set device data size */
619 src_bus_width = stm32_mdma_get_width(chan, src_addr_width);
620 if (src_bus_width < 0)
621 return src_bus_width;
622 ctcr &= ~STM32_MDMA_CTCR_SSIZE_MASK;
623 ctcr |= STM32_MDMA_CTCR_SSIZE(src_bus_width);
624
625 /* Set device burst value */
626 src_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
627 src_maxburst,
628 src_addr_width);
629 ctcr &= ~STM32_MDMA_CTCR_SBURST_MASK;
630 ctcr |= STM32_MDMA_CTCR_SBURST((ilog2(src_best_burst)));
631
632 /* Set memory data size */
633 dst_addr_width = stm32_mdma_get_max_width(addr, buf_len, tlen);
634 chan->mem_width = dst_addr_width;
635 dst_bus_width = stm32_mdma_get_width(chan, dst_addr_width);
636 if (dst_bus_width < 0)
637 return dst_bus_width;
638 ctcr &= ~(STM32_MDMA_CTCR_DSIZE_MASK |
639 STM32_MDMA_CTCR_DINCOS_MASK);
640 ctcr |= STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
641 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
642
643 /* Set memory burst value */
644 dst_maxburst = STM32_MDMA_MAX_BUF_LEN / dst_addr_width;
645 dst_best_burst = stm32_mdma_get_best_burst(buf_len, tlen,
646 dst_maxburst,
647 dst_addr_width);
648 ctcr &= ~STM32_MDMA_CTCR_DBURST_MASK;
649 ctcr |= STM32_MDMA_CTCR_DBURST((ilog2(dst_best_burst)));
650
651 /* Select bus */
652 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
653 src_addr);
654
655 if (dst_bus_width != src_bus_width)
656 ctcr |= STM32_MDMA_CTCR_PKE;
657
658 /* Set source address */
659 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(chan->id), src_addr);
660 break;
661
662 default:
663 dev_err(chan2dev(chan), "Dma direction is not supported\n");
664 return -EINVAL;
665 }
666
667 *mdma_ccr = ccr;
668 *mdma_ctcr = ctcr;
669 *mdma_ctbr = ctbr;
670
671 return 0;
672 }
673
674 static void stm32_mdma_dump_hwdesc(struct stm32_mdma_chan *chan,
675 struct stm32_mdma_desc_node *node)
676 {
677 dev_dbg(chan2dev(chan), "hwdesc: %pad\n", &node->hwdesc_phys);
678 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n", node->hwdesc->ctcr);
679 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n", node->hwdesc->cbndtr);
680 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n", node->hwdesc->csar);
681 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n", node->hwdesc->cdar);
682 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n", node->hwdesc->cbrur);
683 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n", node->hwdesc->clar);
684 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n", node->hwdesc->ctbr);
685 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n", node->hwdesc->cmar);
686 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n\n", node->hwdesc->cmdr);
687 }
688
689 static void stm32_mdma_setup_hwdesc(struct stm32_mdma_chan *chan,
690 struct stm32_mdma_desc *desc,
691 enum dma_transfer_direction dir, u32 count,
692 dma_addr_t src_addr, dma_addr_t dst_addr,
693 u32 len, u32 ctcr, u32 ctbr, bool is_last,
694 bool is_first, bool is_cyclic)
695 {
696 struct stm32_mdma_chan_config *config = &chan->chan_config;
697 struct stm32_mdma_hwdesc *hwdesc;
698 u32 next = count + 1;
699
700 hwdesc = desc->node[count].hwdesc;
701 hwdesc->ctcr = ctcr;
702 hwdesc->cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK |
703 STM32_MDMA_CBNDTR_BRDUM |
704 STM32_MDMA_CBNDTR_BRSUM |
705 STM32_MDMA_CBNDTR_BNDT_MASK);
706 hwdesc->cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
707 hwdesc->csar = src_addr;
708 hwdesc->cdar = dst_addr;
709 hwdesc->cbrur = 0;
710 hwdesc->ctbr = ctbr;
711 hwdesc->cmar = config->mask_addr;
712 hwdesc->cmdr = config->mask_data;
713
714 if (is_last) {
715 if (is_cyclic)
716 hwdesc->clar = desc->node[0].hwdesc_phys;
717 else
718 hwdesc->clar = 0;
719 } else {
720 hwdesc->clar = desc->node[next].hwdesc_phys;
721 }
722
723 stm32_mdma_dump_hwdesc(chan, &desc->node[count]);
724 }
725
726 static int stm32_mdma_setup_xfer(struct stm32_mdma_chan *chan,
727 struct stm32_mdma_desc *desc,
728 struct scatterlist *sgl, u32 sg_len,
729 enum dma_transfer_direction direction)
730 {
731 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
732 struct dma_slave_config *dma_config = &chan->dma_config;
733 struct scatterlist *sg;
734 dma_addr_t src_addr, dst_addr;
735 u32 ccr, ctcr, ctbr;
736 int i, ret = 0;
737
738 for_each_sg(sgl, sg, sg_len, i) {
739 if (sg_dma_len(sg) > STM32_MDMA_MAX_BLOCK_LEN) {
740 dev_err(chan2dev(chan), "Invalid block len\n");
741 return -EINVAL;
742 }
743
744 if (direction == DMA_MEM_TO_DEV) {
745 src_addr = sg_dma_address(sg);
746 dst_addr = dma_config->dst_addr;
747 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
748 &ctcr, &ctbr, src_addr,
749 sg_dma_len(sg));
750 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
751 src_addr);
752 } else {
753 src_addr = dma_config->src_addr;
754 dst_addr = sg_dma_address(sg);
755 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr,
756 &ctcr, &ctbr, dst_addr,
757 sg_dma_len(sg));
758 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
759 dst_addr);
760 }
761
762 if (ret < 0)
763 return ret;
764
765 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,
766 dst_addr, sg_dma_len(sg), ctcr, ctbr,
767 i == sg_len - 1, i == 0, false);
768 }
769
770 /* Enable interrupts */
771 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
772 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE;
773 if (sg_len > 1)
774 ccr |= STM32_MDMA_CCR_BTIE;
775 desc->ccr = ccr;
776
777 return 0;
778 }
779
780 static struct dma_async_tx_descriptor *
781 stm32_mdma_prep_slave_sg(struct dma_chan *c, struct scatterlist *sgl,
782 u32 sg_len, enum dma_transfer_direction direction,
783 unsigned long flags, void *context)
784 {
785 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
786 struct stm32_mdma_desc *desc;
787 int i, ret;
788
789 /*
790 * Once DMA is in setup cyclic mode the channel we cannot assign this
791 * channel anymore. The DMA channel needs to be aborted or terminated
792 * for allowing another request.
793 */
794 if (chan->desc && chan->desc->cyclic) {
795 dev_err(chan2dev(chan),
796 "Request not allowed when dma in cyclic mode\n");
797 return NULL;
798 }
799
800 desc = stm32_mdma_alloc_desc(chan, sg_len);
801 if (!desc)
802 return NULL;
803
804 ret = stm32_mdma_setup_xfer(chan, desc, sgl, sg_len, direction);
805 if (ret < 0)
806 goto xfer_setup_err;
807
808 desc->cyclic = false;
809
810 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
811
812 xfer_setup_err:
813 for (i = 0; i < desc->count; i++)
814 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
815 desc->node[i].hwdesc_phys);
816 kfree(desc);
817 return NULL;
818 }
819
820 static struct dma_async_tx_descriptor *
821 stm32_mdma_prep_dma_cyclic(struct dma_chan *c, dma_addr_t buf_addr,
822 size_t buf_len, size_t period_len,
823 enum dma_transfer_direction direction,
824 unsigned long flags)
825 {
826 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
827 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
828 struct dma_slave_config *dma_config = &chan->dma_config;
829 struct stm32_mdma_desc *desc;
830 dma_addr_t src_addr, dst_addr;
831 u32 ccr, ctcr, ctbr, count;
832 int i, ret;
833
834 /*
835 * Once DMA is in setup cyclic mode the channel we cannot assign this
836 * channel anymore. The DMA channel needs to be aborted or terminated
837 * for allowing another request.
838 */
839 if (chan->desc && chan->desc->cyclic) {
840 dev_err(chan2dev(chan),
841 "Request not allowed when dma in cyclic mode\n");
842 return NULL;
843 }
844
845 if (!buf_len || !period_len || period_len > STM32_MDMA_MAX_BLOCK_LEN) {
846 dev_err(chan2dev(chan), "Invalid buffer/period len\n");
847 return NULL;
848 }
849
850 if (buf_len % period_len) {
851 dev_err(chan2dev(chan), "buf_len not multiple of period_len\n");
852 return NULL;
853 }
854
855 count = buf_len / period_len;
856
857 desc = stm32_mdma_alloc_desc(chan, count);
858 if (!desc)
859 return NULL;
860
861 /* Select bus */
862 if (direction == DMA_MEM_TO_DEV) {
863 src_addr = buf_addr;
864 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,
865 &ctbr, src_addr, period_len);
866 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS,
867 src_addr);
868 } else {
869 dst_addr = buf_addr;
870 ret = stm32_mdma_set_xfer_param(chan, direction, &ccr, &ctcr,
871 &ctbr, dst_addr, period_len);
872 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS,
873 dst_addr);
874 }
875
876 if (ret < 0)
877 goto xfer_setup_err;
878
879 /* Enable interrupts */
880 ccr &= ~STM32_MDMA_CCR_IRQ_MASK;
881 ccr |= STM32_MDMA_CCR_TEIE | STM32_MDMA_CCR_CTCIE | STM32_MDMA_CCR_BTIE;
882 desc->ccr = ccr;
883
884 /* Configure hwdesc list */
885 for (i = 0; i < count; i++) {
886 if (direction == DMA_MEM_TO_DEV) {
887 src_addr = buf_addr + i * period_len;
888 dst_addr = dma_config->dst_addr;
889 } else {
890 src_addr = dma_config->src_addr;
891 dst_addr = buf_addr + i * period_len;
892 }
893
894 stm32_mdma_setup_hwdesc(chan, desc, direction, i, src_addr,
895 dst_addr, period_len, ctcr, ctbr,
896 i == count - 1, i == 0, true);
897 }
898
899 desc->cyclic = true;
900
901 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
902
903 xfer_setup_err:
904 for (i = 0; i < desc->count; i++)
905 dma_pool_free(chan->desc_pool, desc->node[i].hwdesc,
906 desc->node[i].hwdesc_phys);
907 kfree(desc);
908 return NULL;
909 }
910
911 static struct dma_async_tx_descriptor *
912 stm32_mdma_prep_dma_memcpy(struct dma_chan *c, dma_addr_t dest, dma_addr_t src,
913 size_t len, unsigned long flags)
914 {
915 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
916 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
917 enum dma_slave_buswidth max_width;
918 struct stm32_mdma_desc *desc;
919 struct stm32_mdma_hwdesc *hwdesc;
920 u32 ccr, ctcr, ctbr, cbndtr, count, max_burst, mdma_burst;
921 u32 best_burst, tlen;
922 size_t xfer_count, offset;
923 int src_bus_width, dst_bus_width;
924 int i;
925
926 /*
927 * Once DMA is in setup cyclic mode the channel we cannot assign this
928 * channel anymore. The DMA channel needs to be aborted or terminated
929 * to allow another request
930 */
931 if (chan->desc && chan->desc->cyclic) {
932 dev_err(chan2dev(chan),
933 "Request not allowed when dma in cyclic mode\n");
934 return NULL;
935 }
936
937 count = DIV_ROUND_UP(len, STM32_MDMA_MAX_BLOCK_LEN);
938 desc = stm32_mdma_alloc_desc(chan, count);
939 if (!desc)
940 return NULL;
941
942 ccr = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
943 ctcr = stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id));
944 ctbr = stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id));
945 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
946
947 /* Enable sw req, some interrupts and clear other bits */
948 ccr &= ~(STM32_MDMA_CCR_WEX | STM32_MDMA_CCR_HEX |
949 STM32_MDMA_CCR_BEX | STM32_MDMA_CCR_PL_MASK |
950 STM32_MDMA_CCR_IRQ_MASK);
951 ccr |= STM32_MDMA_CCR_TEIE;
952
953 /* Enable SW request mode, dest/src inc and clear other bits */
954 ctcr &= ~(STM32_MDMA_CTCR_BWM | STM32_MDMA_CTCR_TRGM_MSK |
955 STM32_MDMA_CTCR_PAM_MASK | STM32_MDMA_CTCR_PKE |
956 STM32_MDMA_CTCR_TLEN_MSK | STM32_MDMA_CTCR_DBURST_MASK |
957 STM32_MDMA_CTCR_SBURST_MASK | STM32_MDMA_CTCR_DINCOS_MASK |
958 STM32_MDMA_CTCR_SINCOS_MASK | STM32_MDMA_CTCR_DSIZE_MASK |
959 STM32_MDMA_CTCR_SSIZE_MASK | STM32_MDMA_CTCR_DINC_MASK |
960 STM32_MDMA_CTCR_SINC_MASK);
961 ctcr |= STM32_MDMA_CTCR_SWRM | STM32_MDMA_CTCR_SINC(STM32_MDMA_INC) |
962 STM32_MDMA_CTCR_DINC(STM32_MDMA_INC);
963
964 /* Reset HW request */
965 ctbr &= ~STM32_MDMA_CTBR_TSEL_MASK;
966
967 /* Select bus */
968 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_SBUS, src);
969 stm32_mdma_set_bus(dmadev, &ctbr, STM32_MDMA_CTBR_DBUS, dest);
970
971 /* Clear CBNDTR registers */
972 cbndtr &= ~(STM32_MDMA_CBNDTR_BRC_MK | STM32_MDMA_CBNDTR_BRDUM |
973 STM32_MDMA_CBNDTR_BRSUM | STM32_MDMA_CBNDTR_BNDT_MASK);
974
975 if (len <= STM32_MDMA_MAX_BLOCK_LEN) {
976 cbndtr |= STM32_MDMA_CBNDTR_BNDT(len);
977 if (len <= STM32_MDMA_MAX_BUF_LEN) {
978 /* Setup a buffer transfer */
979 ccr |= STM32_MDMA_CCR_TCIE | STM32_MDMA_CCR_CTCIE;
980 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BUFFER);
981 } else {
982 /* Setup a block transfer */
983 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
984 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_BLOCK);
985 }
986
987 tlen = STM32_MDMA_MAX_BUF_LEN;
988 ctcr |= STM32_MDMA_CTCR_TLEN((tlen - 1));
989
990 /* Set source best burst size */
991 max_width = stm32_mdma_get_max_width(src, len, tlen);
992 src_bus_width = stm32_mdma_get_width(chan, max_width);
993
994 max_burst = tlen / max_width;
995 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,
996 max_width);
997 mdma_burst = ilog2(best_burst);
998
999 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1000 STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1001 STM32_MDMA_CTCR_SINCOS(src_bus_width);
1002
1003 /* Set destination best burst size */
1004 max_width = stm32_mdma_get_max_width(dest, len, tlen);
1005 dst_bus_width = stm32_mdma_get_width(chan, max_width);
1006
1007 max_burst = tlen / max_width;
1008 best_burst = stm32_mdma_get_best_burst(len, tlen, max_burst,
1009 max_width);
1010 mdma_burst = ilog2(best_burst);
1011
1012 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1013 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1014 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1015
1016 if (dst_bus_width != src_bus_width)
1017 ctcr |= STM32_MDMA_CTCR_PKE;
1018
1019 /* Prepare hardware descriptor */
1020 hwdesc = desc->node[0].hwdesc;
1021 hwdesc->ctcr = ctcr;
1022 hwdesc->cbndtr = cbndtr;
1023 hwdesc->csar = src;
1024 hwdesc->cdar = dest;
1025 hwdesc->cbrur = 0;
1026 hwdesc->clar = 0;
1027 hwdesc->ctbr = ctbr;
1028 hwdesc->cmar = 0;
1029 hwdesc->cmdr = 0;
1030
1031 stm32_mdma_dump_hwdesc(chan, &desc->node[0]);
1032 } else {
1033 /* Setup a LLI transfer */
1034 ctcr |= STM32_MDMA_CTCR_TRGM(STM32_MDMA_LINKED_LIST) |
1035 STM32_MDMA_CTCR_TLEN((STM32_MDMA_MAX_BUF_LEN - 1));
1036 ccr |= STM32_MDMA_CCR_BTIE | STM32_MDMA_CCR_CTCIE;
1037 tlen = STM32_MDMA_MAX_BUF_LEN;
1038
1039 for (i = 0, offset = 0; offset < len;
1040 i++, offset += xfer_count) {
1041 xfer_count = min_t(size_t, len - offset,
1042 STM32_MDMA_MAX_BLOCK_LEN);
1043
1044 /* Set source best burst size */
1045 max_width = stm32_mdma_get_max_width(src, len, tlen);
1046 src_bus_width = stm32_mdma_get_width(chan, max_width);
1047
1048 max_burst = tlen / max_width;
1049 best_burst = stm32_mdma_get_best_burst(len, tlen,
1050 max_burst,
1051 max_width);
1052 mdma_burst = ilog2(best_burst);
1053
1054 ctcr |= STM32_MDMA_CTCR_SBURST(mdma_burst) |
1055 STM32_MDMA_CTCR_SSIZE(src_bus_width) |
1056 STM32_MDMA_CTCR_SINCOS(src_bus_width);
1057
1058 /* Set destination best burst size */
1059 max_width = stm32_mdma_get_max_width(dest, len, tlen);
1060 dst_bus_width = stm32_mdma_get_width(chan, max_width);
1061
1062 max_burst = tlen / max_width;
1063 best_burst = stm32_mdma_get_best_burst(len, tlen,
1064 max_burst,
1065 max_width);
1066 mdma_burst = ilog2(best_burst);
1067
1068 ctcr |= STM32_MDMA_CTCR_DBURST(mdma_burst) |
1069 STM32_MDMA_CTCR_DSIZE(dst_bus_width) |
1070 STM32_MDMA_CTCR_DINCOS(dst_bus_width);
1071
1072 if (dst_bus_width != src_bus_width)
1073 ctcr |= STM32_MDMA_CTCR_PKE;
1074
1075 /* Prepare hardware descriptor */
1076 stm32_mdma_setup_hwdesc(chan, desc, DMA_MEM_TO_MEM, i,
1077 src + offset, dest + offset,
1078 xfer_count, ctcr, ctbr,
1079 i == count - 1, i == 0, false);
1080 }
1081 }
1082
1083 desc->ccr = ccr;
1084
1085 desc->cyclic = false;
1086
1087 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags);
1088 }
1089
1090 static void stm32_mdma_dump_reg(struct stm32_mdma_chan *chan)
1091 {
1092 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1093
1094 dev_dbg(chan2dev(chan), "CCR: 0x%08x\n",
1095 stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id)));
1096 dev_dbg(chan2dev(chan), "CTCR: 0x%08x\n",
1097 stm32_mdma_read(dmadev, STM32_MDMA_CTCR(chan->id)));
1098 dev_dbg(chan2dev(chan), "CBNDTR: 0x%08x\n",
1099 stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id)));
1100 dev_dbg(chan2dev(chan), "CSAR: 0x%08x\n",
1101 stm32_mdma_read(dmadev, STM32_MDMA_CSAR(chan->id)));
1102 dev_dbg(chan2dev(chan), "CDAR: 0x%08x\n",
1103 stm32_mdma_read(dmadev, STM32_MDMA_CDAR(chan->id)));
1104 dev_dbg(chan2dev(chan), "CBRUR: 0x%08x\n",
1105 stm32_mdma_read(dmadev, STM32_MDMA_CBRUR(chan->id)));
1106 dev_dbg(chan2dev(chan), "CLAR: 0x%08x\n",
1107 stm32_mdma_read(dmadev, STM32_MDMA_CLAR(chan->id)));
1108 dev_dbg(chan2dev(chan), "CTBR: 0x%08x\n",
1109 stm32_mdma_read(dmadev, STM32_MDMA_CTBR(chan->id)));
1110 dev_dbg(chan2dev(chan), "CMAR: 0x%08x\n",
1111 stm32_mdma_read(dmadev, STM32_MDMA_CMAR(chan->id)));
1112 dev_dbg(chan2dev(chan), "CMDR: 0x%08x\n",
1113 stm32_mdma_read(dmadev, STM32_MDMA_CMDR(chan->id)));
1114 }
1115
1116 static void stm32_mdma_start_transfer(struct stm32_mdma_chan *chan)
1117 {
1118 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1119 struct virt_dma_desc *vdesc;
1120 struct stm32_mdma_hwdesc *hwdesc;
1121 u32 id = chan->id;
1122 u32 status, reg;
1123
1124 vdesc = vchan_next_desc(&chan->vchan);
1125 if (!vdesc) {
1126 chan->desc = NULL;
1127 return;
1128 }
1129
1130 chan->desc = to_stm32_mdma_desc(vdesc);
1131 hwdesc = chan->desc->node[0].hwdesc;
1132 chan->curr_hwdesc = 0;
1133
1134 stm32_mdma_write(dmadev, STM32_MDMA_CCR(id), chan->desc->ccr);
1135 stm32_mdma_write(dmadev, STM32_MDMA_CTCR(id), hwdesc->ctcr);
1136 stm32_mdma_write(dmadev, STM32_MDMA_CBNDTR(id), hwdesc->cbndtr);
1137 stm32_mdma_write(dmadev, STM32_MDMA_CSAR(id), hwdesc->csar);
1138 stm32_mdma_write(dmadev, STM32_MDMA_CDAR(id), hwdesc->cdar);
1139 stm32_mdma_write(dmadev, STM32_MDMA_CBRUR(id), hwdesc->cbrur);
1140 stm32_mdma_write(dmadev, STM32_MDMA_CLAR(id), hwdesc->clar);
1141 stm32_mdma_write(dmadev, STM32_MDMA_CTBR(id), hwdesc->ctbr);
1142 stm32_mdma_write(dmadev, STM32_MDMA_CMAR(id), hwdesc->cmar);
1143 stm32_mdma_write(dmadev, STM32_MDMA_CMDR(id), hwdesc->cmdr);
1144
1145 /* Clear interrupt status if it is there */
1146 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(id));
1147 if (status)
1148 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(id), status);
1149
1150 stm32_mdma_dump_reg(chan);
1151
1152 /* Start DMA */
1153 stm32_mdma_set_bits(dmadev, STM32_MDMA_CCR(id), STM32_MDMA_CCR_EN);
1154
1155 /* Set SW request in case of MEM2MEM transfer */
1156 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM) {
1157 reg = STM32_MDMA_CCR(id);
1158 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1159 }
1160
1161 chan->busy = true;
1162
1163 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
1164 }
1165
1166 static void stm32_mdma_issue_pending(struct dma_chan *c)
1167 {
1168 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1169 unsigned long flags;
1170
1171 spin_lock_irqsave(&chan->vchan.lock, flags);
1172
1173 if (!vchan_issue_pending(&chan->vchan))
1174 goto end;
1175
1176 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1177
1178 if (!chan->desc && !chan->busy)
1179 stm32_mdma_start_transfer(chan);
1180
1181 end:
1182 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1183 }
1184
1185 static int stm32_mdma_pause(struct dma_chan *c)
1186 {
1187 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1188 unsigned long flags;
1189 int ret;
1190
1191 spin_lock_irqsave(&chan->vchan.lock, flags);
1192 ret = stm32_mdma_disable_chan(chan);
1193 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1194
1195 if (!ret)
1196 dev_dbg(chan2dev(chan), "vchan %pK: pause\n", &chan->vchan);
1197
1198 return ret;
1199 }
1200
1201 static int stm32_mdma_resume(struct dma_chan *c)
1202 {
1203 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1204 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1205 struct stm32_mdma_hwdesc *hwdesc;
1206 unsigned long flags;
1207 u32 status, reg;
1208
1209 hwdesc = chan->desc->node[chan->curr_hwdesc].hwdesc;
1210
1211 spin_lock_irqsave(&chan->vchan.lock, flags);
1212
1213 /* Re-configure control register */
1214 stm32_mdma_write(dmadev, STM32_MDMA_CCR(chan->id), chan->desc->ccr);
1215
1216 /* Clear interrupt status if it is there */
1217 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1218 if (status)
1219 stm32_mdma_set_bits(dmadev, STM32_MDMA_CIFCR(chan->id), status);
1220
1221 stm32_mdma_dump_reg(chan);
1222
1223 /* Re-start DMA */
1224 reg = STM32_MDMA_CCR(chan->id);
1225 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_EN);
1226
1227 /* Set SW request in case of MEM2MEM transfer */
1228 if (hwdesc->ctcr & STM32_MDMA_CTCR_SWRM)
1229 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CCR_SWRQ);
1230
1231 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1232
1233 dev_dbg(chan2dev(chan), "vchan %pK: resume\n", &chan->vchan);
1234
1235 return 0;
1236 }
1237
1238 static int stm32_mdma_terminate_all(struct dma_chan *c)
1239 {
1240 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1241 unsigned long flags;
1242 LIST_HEAD(head);
1243
1244 spin_lock_irqsave(&chan->vchan.lock, flags);
1245 if (chan->busy) {
1246 stm32_mdma_stop(chan);
1247 chan->desc = NULL;
1248 }
1249 vchan_get_all_descriptors(&chan->vchan, &head);
1250 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1251
1252 vchan_dma_desc_free_list(&chan->vchan, &head);
1253
1254 return 0;
1255 }
1256
1257 static void stm32_mdma_synchronize(struct dma_chan *c)
1258 {
1259 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1260
1261 vchan_synchronize(&chan->vchan);
1262 }
1263
1264 static int stm32_mdma_slave_config(struct dma_chan *c,
1265 struct dma_slave_config *config)
1266 {
1267 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1268
1269 memcpy(&chan->dma_config, config, sizeof(*config));
1270
1271 return 0;
1272 }
1273
1274 static size_t stm32_mdma_desc_residue(struct stm32_mdma_chan *chan,
1275 struct stm32_mdma_desc *desc,
1276 u32 curr_hwdesc)
1277 {
1278 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1279 struct stm32_mdma_hwdesc *hwdesc = desc->node[0].hwdesc;
1280 u32 cbndtr, residue, modulo, burst_size;
1281 int i;
1282
1283 residue = 0;
1284 for (i = curr_hwdesc + 1; i < desc->count; i++) {
1285 hwdesc = desc->node[i].hwdesc;
1286 residue += STM32_MDMA_CBNDTR_BNDT(hwdesc->cbndtr);
1287 }
1288 cbndtr = stm32_mdma_read(dmadev, STM32_MDMA_CBNDTR(chan->id));
1289 residue += cbndtr & STM32_MDMA_CBNDTR_BNDT_MASK;
1290
1291 if (!chan->mem_burst)
1292 return residue;
1293
1294 burst_size = chan->mem_burst * chan->mem_width;
1295 modulo = residue % burst_size;
1296 if (modulo)
1297 residue = residue - modulo + burst_size;
1298
1299 return residue;
1300 }
1301
1302 static enum dma_status stm32_mdma_tx_status(struct dma_chan *c,
1303 dma_cookie_t cookie,
1304 struct dma_tx_state *state)
1305 {
1306 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1307 struct virt_dma_desc *vdesc;
1308 enum dma_status status;
1309 unsigned long flags;
1310 u32 residue = 0;
1311
1312 status = dma_cookie_status(c, cookie, state);
1313 if ((status == DMA_COMPLETE) || (!state))
1314 return status;
1315
1316 spin_lock_irqsave(&chan->vchan.lock, flags);
1317
1318 vdesc = vchan_find_desc(&chan->vchan, cookie);
1319 if (chan->desc && cookie == chan->desc->vdesc.tx.cookie)
1320 residue = stm32_mdma_desc_residue(chan, chan->desc,
1321 chan->curr_hwdesc);
1322 else if (vdesc)
1323 residue = stm32_mdma_desc_residue(chan,
1324 to_stm32_mdma_desc(vdesc), 0);
1325 dma_set_residue(state, residue);
1326
1327 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1328
1329 return status;
1330 }
1331
1332 static void stm32_mdma_xfer_end(struct stm32_mdma_chan *chan)
1333 {
1334 list_del(&chan->desc->vdesc.node);
1335 vchan_cookie_complete(&chan->desc->vdesc);
1336 chan->desc = NULL;
1337 chan->busy = false;
1338
1339 /* Start the next transfer if this driver has a next desc */
1340 stm32_mdma_start_transfer(chan);
1341 }
1342
1343 static irqreturn_t stm32_mdma_irq_handler(int irq, void *devid)
1344 {
1345 struct stm32_mdma_device *dmadev = devid;
1346 struct stm32_mdma_chan *chan = devid;
1347 u32 reg, id, ien, status, flag;
1348
1349 /* Find out which channel generates the interrupt */
1350 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR0);
1351 if (status) {
1352 id = __ffs(status);
1353 } else {
1354 status = readl_relaxed(dmadev->base + STM32_MDMA_GISR1);
1355 if (!status) {
1356 dev_dbg(mdma2dev(dmadev), "spurious it\n");
1357 return IRQ_NONE;
1358 }
1359 id = __ffs(status);
1360 /*
1361 * As GISR0 provides status for channel id from 0 to 31,
1362 * so GISR1 provides status for channel id from 32 to 62
1363 */
1364 id += 32;
1365 }
1366
1367 chan = &dmadev->chan[id];
1368 if (!chan) {
1369 dev_dbg(mdma2dev(dmadev), "MDMA channel not initialized\n");
1370 goto exit;
1371 }
1372
1373 /* Handle interrupt for the channel */
1374 spin_lock(&chan->vchan.lock);
1375 status = stm32_mdma_read(dmadev, STM32_MDMA_CISR(chan->id));
1376 ien = stm32_mdma_read(dmadev, STM32_MDMA_CCR(chan->id));
1377 ien &= STM32_MDMA_CCR_IRQ_MASK;
1378 ien >>= 1;
1379
1380 if (!(status & ien)) {
1381 spin_unlock(&chan->vchan.lock);
1382 dev_dbg(chan2dev(chan),
1383 "spurious it (status=0x%04x, ien=0x%04x)\n",
1384 status, ien);
1385 return IRQ_NONE;
1386 }
1387
1388 flag = __ffs(status & ien);
1389 reg = STM32_MDMA_CIFCR(chan->id);
1390
1391 switch (1 << flag) {
1392 case STM32_MDMA_CISR_TEIF:
1393 id = chan->id;
1394 status = readl_relaxed(dmadev->base + STM32_MDMA_CESR(id));
1395 dev_err(chan2dev(chan), "Transfer Err: stat=0x%08x\n", status);
1396 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CTEIF);
1397 break;
1398
1399 case STM32_MDMA_CISR_CTCIF:
1400 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CCTCIF);
1401 stm32_mdma_xfer_end(chan);
1402 break;
1403
1404 case STM32_MDMA_CISR_BRTIF:
1405 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBRTIF);
1406 break;
1407
1408 case STM32_MDMA_CISR_BTIF:
1409 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CBTIF);
1410 chan->curr_hwdesc++;
1411 if (chan->desc && chan->desc->cyclic) {
1412 if (chan->curr_hwdesc == chan->desc->count)
1413 chan->curr_hwdesc = 0;
1414 vchan_cyclic_callback(&chan->desc->vdesc);
1415 }
1416 break;
1417
1418 case STM32_MDMA_CISR_TCIF:
1419 stm32_mdma_set_bits(dmadev, reg, STM32_MDMA_CIFCR_CLTCIF);
1420 break;
1421
1422 default:
1423 dev_err(chan2dev(chan), "it %d unhandled (status=0x%04x)\n",
1424 1 << flag, status);
1425 }
1426
1427 spin_unlock(&chan->vchan.lock);
1428
1429 exit:
1430 return IRQ_HANDLED;
1431 }
1432
1433 static int stm32_mdma_alloc_chan_resources(struct dma_chan *c)
1434 {
1435 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1436 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1437 int ret;
1438
1439 chan->desc_pool = dmam_pool_create(dev_name(&c->dev->device),
1440 c->device->dev,
1441 sizeof(struct stm32_mdma_hwdesc),
1442 __alignof__(struct stm32_mdma_hwdesc),
1443 0);
1444 if (!chan->desc_pool) {
1445 dev_err(chan2dev(chan), "failed to allocate descriptor pool\n");
1446 return -ENOMEM;
1447 }
1448
1449 ret = pm_runtime_get_sync(dmadev->ddev.dev);
1450 if (ret < 0)
1451 return ret;
1452
1453 ret = stm32_mdma_disable_chan(chan);
1454 if (ret < 0)
1455 pm_runtime_put(dmadev->ddev.dev);
1456
1457 return ret;
1458 }
1459
1460 static void stm32_mdma_free_chan_resources(struct dma_chan *c)
1461 {
1462 struct stm32_mdma_chan *chan = to_stm32_mdma_chan(c);
1463 struct stm32_mdma_device *dmadev = stm32_mdma_get_dev(chan);
1464 unsigned long flags;
1465
1466 dev_dbg(chan2dev(chan), "Freeing channel %d\n", chan->id);
1467
1468 if (chan->busy) {
1469 spin_lock_irqsave(&chan->vchan.lock, flags);
1470 stm32_mdma_stop(chan);
1471 chan->desc = NULL;
1472 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1473 }
1474
1475 pm_runtime_put(dmadev->ddev.dev);
1476 vchan_free_chan_resources(to_virt_chan(c));
1477 dmam_pool_destroy(chan->desc_pool);
1478 chan->desc_pool = NULL;
1479 }
1480
1481 static struct dma_chan *stm32_mdma_of_xlate(struct of_phandle_args *dma_spec,
1482 struct of_dma *ofdma)
1483 {
1484 struct stm32_mdma_device *dmadev = ofdma->of_dma_data;
1485 struct stm32_mdma_chan *chan;
1486 struct dma_chan *c;
1487 struct stm32_mdma_chan_config config;
1488
1489 if (dma_spec->args_count < 5) {
1490 dev_err(mdma2dev(dmadev), "Bad number of args\n");
1491 return NULL;
1492 }
1493
1494 config.request = dma_spec->args[0];
1495 config.priority_level = dma_spec->args[1];
1496 config.transfer_config = dma_spec->args[2];
1497 config.mask_addr = dma_spec->args[3];
1498 config.mask_data = dma_spec->args[4];
1499
1500 if (config.request >= dmadev->nr_requests) {
1501 dev_err(mdma2dev(dmadev), "Bad request line\n");
1502 return NULL;
1503 }
1504
1505 if (config.priority_level > STM32_MDMA_VERY_HIGH_PRIORITY) {
1506 dev_err(mdma2dev(dmadev), "Priority level not supported\n");
1507 return NULL;
1508 }
1509
1510 c = dma_get_any_slave_channel(&dmadev->ddev);
1511 if (!c) {
1512 dev_err(mdma2dev(dmadev), "No more channels available\n");
1513 return NULL;
1514 }
1515
1516 chan = to_stm32_mdma_chan(c);
1517 chan->chan_config = config;
1518
1519 return c;
1520 }
1521
1522 static const struct of_device_id stm32_mdma_of_match[] = {
1523 { .compatible = "st,stm32h7-mdma", },
1524 { /* sentinel */ },
1525 };
1526 MODULE_DEVICE_TABLE(of, stm32_mdma_of_match);
1527
1528 static int stm32_mdma_probe(struct platform_device *pdev)
1529 {
1530 struct stm32_mdma_chan *chan;
1531 struct stm32_mdma_device *dmadev;
1532 struct dma_device *dd;
1533 struct device_node *of_node;
1534 struct resource *res;
1535 u32 nr_channels, nr_requests;
1536 int i, count, ret;
1537
1538 of_node = pdev->dev.of_node;
1539 if (!of_node)
1540 return -ENODEV;
1541
1542 ret = device_property_read_u32(&pdev->dev, "dma-channels",
1543 &nr_channels);
1544 if (ret) {
1545 nr_channels = STM32_MDMA_MAX_CHANNELS;
1546 dev_warn(&pdev->dev, "MDMA defaulting on %i channels\n",
1547 nr_channels);
1548 }
1549
1550 ret = device_property_read_u32(&pdev->dev, "dma-requests",
1551 &nr_requests);
1552 if (ret) {
1553 nr_requests = STM32_MDMA_MAX_REQUESTS;
1554 dev_warn(&pdev->dev, "MDMA defaulting on %i request lines\n",
1555 nr_requests);
1556 }
1557
1558 count = device_property_count_u32(&pdev->dev, "st,ahb-addr-masks");
1559 if (count < 0)
1560 count = 0;
1561
1562 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev) + sizeof(u32) * count,
1563 GFP_KERNEL);
1564 if (!dmadev)
1565 return -ENOMEM;
1566
1567 dmadev->nr_channels = nr_channels;
1568 dmadev->nr_requests = nr_requests;
1569 device_property_read_u32_array(&pdev->dev, "st,ahb-addr-masks",
1570 dmadev->ahb_addr_masks,
1571 count);
1572 dmadev->nr_ahb_addr_masks = count;
1573
1574 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1575 dmadev->base = devm_ioremap_resource(&pdev->dev, res);
1576 if (IS_ERR(dmadev->base))
1577 return PTR_ERR(dmadev->base);
1578
1579 dmadev->clk = devm_clk_get(&pdev->dev, NULL);
1580 if (IS_ERR(dmadev->clk)) {
1581 ret = PTR_ERR(dmadev->clk);
1582 if (ret == -EPROBE_DEFER)
1583 dev_info(&pdev->dev, "Missing controller clock\n");
1584 return ret;
1585 }
1586
1587 ret = clk_prepare_enable(dmadev->clk);
1588 if (ret < 0) {
1589 dev_err(&pdev->dev, "clk_prep_enable error: %d\n", ret);
1590 return ret;
1591 }
1592
1593 dmadev->rst = devm_reset_control_get(&pdev->dev, NULL);
1594 if (!IS_ERR(dmadev->rst)) {
1595 reset_control_assert(dmadev->rst);
1596 udelay(2);
1597 reset_control_deassert(dmadev->rst);
1598 }
1599
1600 dd = &dmadev->ddev;
1601 dma_cap_set(DMA_SLAVE, dd->cap_mask);
1602 dma_cap_set(DMA_PRIVATE, dd->cap_mask);
1603 dma_cap_set(DMA_CYCLIC, dd->cap_mask);
1604 dma_cap_set(DMA_MEMCPY, dd->cap_mask);
1605 dd->device_alloc_chan_resources = stm32_mdma_alloc_chan_resources;
1606 dd->device_free_chan_resources = stm32_mdma_free_chan_resources;
1607 dd->device_tx_status = stm32_mdma_tx_status;
1608 dd->device_issue_pending = stm32_mdma_issue_pending;
1609 dd->device_prep_slave_sg = stm32_mdma_prep_slave_sg;
1610 dd->device_prep_dma_cyclic = stm32_mdma_prep_dma_cyclic;
1611 dd->device_prep_dma_memcpy = stm32_mdma_prep_dma_memcpy;
1612 dd->device_config = stm32_mdma_slave_config;
1613 dd->device_pause = stm32_mdma_pause;
1614 dd->device_resume = stm32_mdma_resume;
1615 dd->device_terminate_all = stm32_mdma_terminate_all;
1616 dd->device_synchronize = stm32_mdma_synchronize;
1617 dd->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1618 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1619 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1620 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1621 dd->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1622 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1623 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1624 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1625 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) |
1626 BIT(DMA_MEM_TO_MEM);
1627 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1628 dd->max_burst = STM32_MDMA_MAX_BURST;
1629 dd->dev = &pdev->dev;
1630 INIT_LIST_HEAD(&dd->channels);
1631
1632 for (i = 0; i < dmadev->nr_channels; i++) {
1633 chan = &dmadev->chan[i];
1634 chan->id = i;
1635 chan->vchan.desc_free = stm32_mdma_desc_free;
1636 vchan_init(&chan->vchan, dd);
1637 }
1638
1639 dmadev->irq = platform_get_irq(pdev, 0);
1640 if (dmadev->irq < 0)
1641 return dmadev->irq;
1642
1643 ret = devm_request_irq(&pdev->dev, dmadev->irq, stm32_mdma_irq_handler,
1644 0, dev_name(&pdev->dev), dmadev);
1645 if (ret) {
1646 dev_err(&pdev->dev, "failed to request IRQ\n");
1647 return ret;
1648 }
1649
1650 ret = dmaenginem_async_device_register(dd);
1651 if (ret)
1652 return ret;
1653
1654 ret = of_dma_controller_register(of_node, stm32_mdma_of_xlate, dmadev);
1655 if (ret < 0) {
1656 dev_err(&pdev->dev,
1657 "STM32 MDMA DMA OF registration failed %d\n", ret);
1658 goto err_unregister;
1659 }
1660
1661 platform_set_drvdata(pdev, dmadev);
1662 pm_runtime_set_active(&pdev->dev);
1663 pm_runtime_enable(&pdev->dev);
1664 pm_runtime_get_noresume(&pdev->dev);
1665 pm_runtime_put(&pdev->dev);
1666
1667 dev_info(&pdev->dev, "STM32 MDMA driver registered\n");
1668
1669 return 0;
1670
1671 err_unregister:
1672 return ret;
1673 }
1674
1675 #ifdef CONFIG_PM
1676 static int stm32_mdma_runtime_suspend(struct device *dev)
1677 {
1678 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1679
1680 clk_disable_unprepare(dmadev->clk);
1681
1682 return 0;
1683 }
1684
1685 static int stm32_mdma_runtime_resume(struct device *dev)
1686 {
1687 struct stm32_mdma_device *dmadev = dev_get_drvdata(dev);
1688 int ret;
1689
1690 ret = clk_prepare_enable(dmadev->clk);
1691 if (ret) {
1692 dev_err(dev, "failed to prepare_enable clock\n");
1693 return ret;
1694 }
1695
1696 return 0;
1697 }
1698 #endif
1699
1700 static const struct dev_pm_ops stm32_mdma_pm_ops = {
1701 SET_RUNTIME_PM_OPS(stm32_mdma_runtime_suspend,
1702 stm32_mdma_runtime_resume, NULL)
1703 };
1704
1705 static struct platform_driver stm32_mdma_driver = {
1706 .probe = stm32_mdma_probe,
1707 .driver = {
1708 .name = "stm32-mdma",
1709 .of_match_table = stm32_mdma_of_match,
1710 .pm = &stm32_mdma_pm_ops,
1711 },
1712 };
1713
1714 static int __init stm32_mdma_init(void)
1715 {
1716 return platform_driver_register(&stm32_mdma_driver);
1717 }
1718
1719 subsys_initcall(stm32_mdma_init);
1720
1721 MODULE_DESCRIPTION("Driver for STM32 MDMA controller");
1722 MODULE_AUTHOR("M'boumba Cedric Madianga <cedric.madianga@gmail.com>");
1723 MODULE_AUTHOR("Pierre-Yves Mordret <pierre-yves.mordret@st.com>");
1724 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support