1 // SPDX-License-Identifier: GPL-2.0-only
3 * STM32 DMA3 controller driver
5 * Copyright (C) STMicroelectronics 2024
6 * Author(s): Amelie Delaunay <amelie.delaunay@foss.st.com>
9 #include <linux/bitfield.h>
10 #include <linux/clk.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmaengine.h>
13 #include <linux/dmapool.h>
14 #include <linux/init.h>
15 #include <linux/iopoll.h>
16 #include <linux/list.h>
17 #include <linux/module.h>
18 #include <linux/of_dma.h>
19 #include <linux/platform_device.h>
20 #include <linux/pm_runtime.h>
21 #include <linux/reset.h>
22 #include <linux/slab.h>
24 #include "../virt-dma.h"
26 #define STM32_DMA3_SECCFGR 0x00
27 #define STM32_DMA3_PRIVCFGR 0x04
28 #define STM32_DMA3_RCFGLOCKR 0x08
29 #define STM32_DMA3_MISR 0x0c
30 #define STM32_DMA3_SMISR 0x10
32 #define STM32_DMA3_CLBAR(x) (0x50 + 0x80 * (x))
33 #define STM32_DMA3_CCIDCFGR(x) (0x54 + 0x80 * (x))
34 #define STM32_DMA3_CSEMCR(x) (0x58 + 0x80 * (x))
35 #define STM32_DMA3_CFCR(x) (0x5c + 0x80 * (x))
36 #define STM32_DMA3_CSR(x) (0x60 + 0x80 * (x))
37 #define STM32_DMA3_CCR(x) (0x64 + 0x80 * (x))
38 #define STM32_DMA3_CTR1(x) (0x90 + 0x80 * (x))
39 #define STM32_DMA3_CTR2(x) (0x94 + 0x80 * (x))
40 #define STM32_DMA3_CBR1(x) (0x98 + 0x80 * (x))
41 #define STM32_DMA3_CSAR(x) (0x9c + 0x80 * (x))
42 #define STM32_DMA3_CDAR(x) (0xa0 + 0x80 * (x))
43 #define STM32_DMA3_CLLR(x) (0xcc + 0x80 * (x))
45 #define STM32_DMA3_HWCFGR13 0xfc0 /* G_PER_CTRL(X) x=8..15 */
46 #define STM32_DMA3_HWCFGR12 0xfc4 /* G_PER_CTRL(X) x=0..7 */
47 #define STM32_DMA3_HWCFGR4 0xfe4 /* G_FIFO_SIZE(X) x=8..15 */
48 #define STM32_DMA3_HWCFGR3 0xfe8 /* G_FIFO_SIZE(X) x=0..7 */
49 #define STM32_DMA3_HWCFGR2 0xfec /* G_MAX_REQ_ID */
50 #define STM32_DMA3_HWCFGR1 0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */
51 #define STM32_DMA3_VERR 0xff4
53 /* SECCFGR DMA secure configuration register */
54 #define SECCFGR_SEC(x) BIT(x)
56 /* MISR DMA non-secure/secure masked interrupt status register */
57 #define MISR_MIS(x) BIT(x)
59 /* CxLBAR DMA channel x linked_list base address register */
60 #define CLBAR_LBA GENMASK(31, 16)
62 /* CxCIDCFGR DMA channel x CID register */
63 #define CCIDCFGR_CFEN BIT(0)
64 #define CCIDCFGR_SEM_EN BIT(1)
65 #define CCIDCFGR_SCID GENMASK(5, 4)
66 #define CCIDCFGR_SEM_WLIST_CID0 BIT(16)
67 #define CCIDCFGR_SEM_WLIST_CID1 BIT(17)
68 #define CCIDCFGR_SEM_WLIST_CID2 BIT(18)
76 /* CxSEMCR DMA channel x semaphore control register */
77 #define CSEMCR_SEM_MUTEX BIT(0)
78 #define CSEMCR_SEM_CCID GENMASK(5, 4)
80 /* CxFCR DMA channel x flag clear register */
81 #define CFCR_TCF BIT(8)
82 #define CFCR_HTF BIT(9)
83 #define CFCR_DTEF BIT(10)
84 #define CFCR_ULEF BIT(11)
85 #define CFCR_USEF BIT(12)
86 #define CFCR_SUSPF BIT(13)
88 /* CxSR DMA channel x status register */
89 #define CSR_IDLEF BIT(0)
90 #define CSR_TCF BIT(8)
91 #define CSR_HTF BIT(9)
92 #define CSR_DTEF BIT(10)
93 #define CSR_ULEF BIT(11)
94 #define CSR_USEF BIT(12)
95 #define CSR_SUSPF BIT(13)
96 #define CSR_ALL_F GENMASK(13, 8)
97 #define CSR_FIFOL GENMASK(24, 16)
99 /* CxCR DMA channel x control register */
100 #define CCR_EN BIT(0)
101 #define CCR_RESET BIT(1)
102 #define CCR_SUSP BIT(2)
103 #define CCR_TCIE BIT(8)
104 #define CCR_HTIE BIT(9)
105 #define CCR_DTEIE BIT(10)
106 #define CCR_ULEIE BIT(11)
107 #define CCR_USEIE BIT(12)
108 #define CCR_SUSPIE BIT(13)
109 #define CCR_ALLIE GENMASK(13, 8)
110 #define CCR_LSM BIT(16)
111 #define CCR_LAP BIT(17)
112 #define CCR_PRIO GENMASK(23, 22)
121 /* CxTR1 DMA channel x transfer register 1 */
122 #define CTR1_SINC BIT(3)
123 #define CTR1_SBL_1 GENMASK(9, 4)
124 #define CTR1_DINC BIT(19)
125 #define CTR1_DBL_1 GENMASK(25, 20)
126 #define CTR1_SDW_LOG2 GENMASK(1, 0)
127 #define CTR1_PAM GENMASK(12, 11)
128 #define CTR1_SAP BIT(14)
129 #define CTR1_DDW_LOG2 GENMASK(17, 16)
130 #define CTR1_DAP BIT(30)
136 CTR1_DW_DWORD
, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */
140 CTR1_PAM_0S_LT
, /* if DDW > SDW, padded with 0s else left-truncated */
141 CTR1_PAM_SE_RT
, /* if DDW > SDW, sign extended else right-truncated */
142 CTR1_PAM_PACK_UNPACK
, /* FIFO queued */
145 /* CxTR2 DMA channel x transfer register 2 */
146 #define CTR2_REQSEL GENMASK(7, 0)
147 #define CTR2_SWREQ BIT(9)
148 #define CTR2_DREQ BIT(10)
149 #define CTR2_BREQ BIT(11)
150 #define CTR2_PFREQ BIT(12)
151 #define CTR2_TCEM GENMASK(31, 30)
155 CTR2_TCEM_REPEAT_BLOCK
,
160 /* CxBR1 DMA channel x block register 1 */
161 #define CBR1_BNDT GENMASK(15, 0)
163 /* CxLLR DMA channel x linked-list address register */
164 #define CLLR_LA GENMASK(15, 2)
165 #define CLLR_ULL BIT(16)
166 #define CLLR_UDA BIT(27)
167 #define CLLR_USA BIT(28)
168 #define CLLR_UB1 BIT(29)
169 #define CLLR_UT2 BIT(30)
170 #define CLLR_UT1 BIT(31)
172 /* HWCFGR13 DMA hardware configuration register 13 x=8..15 */
173 /* HWCFGR12 DMA hardware configuration register 12 x=0..7 */
174 #define G_PER_CTRL(x) (ULL(0x1) << (4 * (x)))
176 /* HWCFGR4 DMA hardware configuration register 4 x=8..15 */
177 /* HWCFGR3 DMA hardware configuration register 3 x=0..7 */
178 #define G_FIFO_SIZE(x) (ULL(0x7) << (4 * (x)))
180 #define get_chan_hwcfg(x, mask, reg) (((reg) & (mask)) >> (4 * (x)))
182 /* HWCFGR2 DMA hardware configuration register 2 */
183 #define G_MAX_REQ_ID GENMASK(7, 0)
185 /* HWCFGR1 DMA hardware configuration register 1 */
186 #define G_MASTER_PORTS GENMASK(2, 0)
187 #define G_NUM_CHANNELS GENMASK(12, 8)
188 #define G_M0_DATA_WIDTH_ENC GENMASK(25, 24)
189 #define G_M1_DATA_WIDTH_ENC GENMASK(29, 28)
191 enum stm32_dma3_master_ports
{
192 AXI64
, /* 1x AXI: 64-bit port 0 */
193 AHB32
, /* 1x AHB: 32-bit port 0 */
194 AHB32_AHB32
, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */
195 AXI64_AHB32
, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */
196 AXI64_AXI64
, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */
197 AXI128_AHB32
, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */
200 enum stm32_dma3_port_data_width
{
201 DW_32
, /* 32-bit, for AHB */
202 DW_64
, /* 64-bit, for AXI */
203 DW_128
, /* 128-bit, for AXI */
207 /* VERR DMA version register */
208 #define VERR_MINREV GENMASK(3, 0)
209 #define VERR_MAJREV GENMASK(7, 4)
212 /* struct stm32_dma3_dt_conf */
214 #define STM32_DMA3_DT_PRIO GENMASK(1, 0) /* CCR_PRIO */
215 #define STM32_DMA3_DT_FIFO GENMASK(7, 4)
217 #define STM32_DMA3_DT_SINC BIT(0) /* CTR1_SINC */
218 #define STM32_DMA3_DT_SAP BIT(1) /* CTR1_SAP */
219 #define STM32_DMA3_DT_DINC BIT(4) /* CTR1_DINC */
220 #define STM32_DMA3_DT_DAP BIT(5) /* CTR1_DAP */
221 #define STM32_DMA3_DT_BREQ BIT(8) /* CTR2_BREQ */
222 #define STM32_DMA3_DT_PFREQ BIT(9) /* CTR2_PFREQ */
223 #define STM32_DMA3_DT_TCEM GENMASK(13, 12) /* CTR2_TCEM */
225 /* struct stm32_dma3_chan .config_set bitfield */
226 #define STM32_DMA3_CFG_SET_DT BIT(0)
227 #define STM32_DMA3_CFG_SET_DMA BIT(1)
228 #define STM32_DMA3_CFG_SET_BOTH (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA)
230 #define STM32_DMA3_MAX_BLOCK_SIZE ALIGN_DOWN(CBR1_BNDT, 64)
231 #define port_is_ahb(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \
232 ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); })
233 #define port_is_axi(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \
234 ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); })
235 #define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) || \
236 (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \
237 DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES)
239 /* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */
240 struct stm32_dma3_hwdesc
{
247 } __packed
__aligned(32);
250 * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed
251 * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte
252 * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents
253 * the sg size limitation.
255 #define STM32_DMA3_MAX_SEG_SIZE ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE)
260 struct stm32_dma3_lli
{
261 struct stm32_dma3_hwdesc
*hwdesc
;
262 dma_addr_t hwdesc_addr
;
265 struct stm32_dma3_swdesc
{
266 struct virt_dma_desc vdesc
;
270 struct stm32_dma3_lli lli
[] __counted_by(lli_size
);
273 struct stm32_dma3_dt_conf
{
280 struct stm32_dma3_chan
{
281 struct virt_dma_chan vchan
;
287 struct stm32_dma3_dt_conf dt_config
;
288 struct dma_slave_config dma_config
;
290 struct dma_pool
*lli_pool
;
291 struct stm32_dma3_swdesc
*swdesc
;
296 struct stm32_dma3_ddata
{
297 struct dma_device dma_dev
;
300 struct stm32_dma3_chan
*chans
;
303 enum stm32_dma3_port_data_width ports_max_dw
[2];
306 static inline struct stm32_dma3_ddata
*to_stm32_dma3_ddata(struct stm32_dma3_chan
*chan
)
308 return container_of(chan
->vchan
.chan
.device
, struct stm32_dma3_ddata
, dma_dev
);
311 static inline struct stm32_dma3_chan
*to_stm32_dma3_chan(struct dma_chan
*c
)
313 return container_of(c
, struct stm32_dma3_chan
, vchan
.chan
);
316 static inline struct stm32_dma3_swdesc
*to_stm32_dma3_swdesc(struct virt_dma_desc
*vdesc
)
318 return container_of(vdesc
, struct stm32_dma3_swdesc
, vdesc
);
321 static struct device
*chan2dev(struct stm32_dma3_chan
*chan
)
323 return &chan
->vchan
.chan
.dev
->device
;
326 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan
*chan
)
328 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
329 struct device
*dev
= chan2dev(chan
);
330 u32 id
= chan
->id
, offset
;
332 offset
= STM32_DMA3_SECCFGR
;
333 dev_dbg(dev
, "SECCFGR(0x%03x): %08x\n", offset
, readl_relaxed(ddata
->base
+ offset
));
334 offset
= STM32_DMA3_PRIVCFGR
;
335 dev_dbg(dev
, "PRIVCFGR(0x%03x): %08x\n", offset
, readl_relaxed(ddata
->base
+ offset
));
336 offset
= STM32_DMA3_CCIDCFGR(id
);
337 dev_dbg(dev
, "C%dCIDCFGR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
338 offset
= STM32_DMA3_CSEMCR(id
);
339 dev_dbg(dev
, "C%dSEMCR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
340 offset
= STM32_DMA3_CSR(id
);
341 dev_dbg(dev
, "C%dSR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
342 offset
= STM32_DMA3_CCR(id
);
343 dev_dbg(dev
, "C%dCR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
344 offset
= STM32_DMA3_CTR1(id
);
345 dev_dbg(dev
, "C%dTR1(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
346 offset
= STM32_DMA3_CTR2(id
);
347 dev_dbg(dev
, "C%dTR2(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
348 offset
= STM32_DMA3_CBR1(id
);
349 dev_dbg(dev
, "C%dBR1(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
350 offset
= STM32_DMA3_CSAR(id
);
351 dev_dbg(dev
, "C%dSAR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
352 offset
= STM32_DMA3_CDAR(id
);
353 dev_dbg(dev
, "C%dDAR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
354 offset
= STM32_DMA3_CLLR(id
);
355 dev_dbg(dev
, "C%dLLR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
356 offset
= STM32_DMA3_CLBAR(id
);
357 dev_dbg(dev
, "C%dLBAR(0x%03x): %08x\n", id
, offset
, readl_relaxed(ddata
->base
+ offset
));
360 static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan
*chan
,
361 struct stm32_dma3_swdesc
*swdesc
)
363 struct stm32_dma3_hwdesc
*hwdesc
;
366 for (i
= 0; i
< swdesc
->lli_size
; i
++) {
367 hwdesc
= swdesc
->lli
[i
].hwdesc
;
369 dev_dbg(chan2dev(chan
), "V\n");
370 dev_dbg(chan2dev(chan
), "[%d]@%pad\n", i
, &swdesc
->lli
[i
].hwdesc_addr
);
371 dev_dbg(chan2dev(chan
), "| C%dTR1: %08x\n", chan
->id
, hwdesc
->ctr1
);
372 dev_dbg(chan2dev(chan
), "| C%dTR2: %08x\n", chan
->id
, hwdesc
->ctr2
);
373 dev_dbg(chan2dev(chan
), "| C%dBR1: %08x\n", chan
->id
, hwdesc
->cbr1
);
374 dev_dbg(chan2dev(chan
), "| C%dSAR: %08x\n", chan
->id
, hwdesc
->csar
);
375 dev_dbg(chan2dev(chan
), "| C%dDAR: %08x\n", chan
->id
, hwdesc
->cdar
);
376 dev_dbg(chan2dev(chan
), "| C%dLLR: %08x\n", chan
->id
, hwdesc
->cllr
);
379 if (swdesc
->cyclic
) {
380 dev_dbg(chan2dev(chan
), "|\n");
381 dev_dbg(chan2dev(chan
), "-->[0]@%pad\n", &swdesc
->lli
[0].hwdesc_addr
);
383 dev_dbg(chan2dev(chan
), "X\n");
387 static struct stm32_dma3_swdesc
*stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan
*chan
, u32 count
)
389 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
390 struct stm32_dma3_swdesc
*swdesc
;
394 * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes
395 * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be
396 * addressed, so abort the allocation.
398 if ((count
* 32) > CLLR_LA
) {
399 dev_err(chan2dev(chan
), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE
);
403 swdesc
= kzalloc(struct_size(swdesc
, lli
, count
), GFP_NOWAIT
);
406 swdesc
->lli_size
= count
;
408 for (i
= 0; i
< count
; i
++) {
409 swdesc
->lli
[i
].hwdesc
= dma_pool_zalloc(chan
->lli_pool
, GFP_NOWAIT
,
410 &swdesc
->lli
[i
].hwdesc_addr
);
411 if (!swdesc
->lli
[i
].hwdesc
)
416 /* Set LL base address */
417 writel_relaxed(swdesc
->lli
[0].hwdesc_addr
& CLBAR_LBA
,
418 ddata
->base
+ STM32_DMA3_CLBAR(chan
->id
));
420 /* Set LL allocated port */
421 swdesc
->ccr
&= ~CCR_LAP
;
426 dev_err(chan2dev(chan
), "Failed to alloc descriptors\n");
428 dma_pool_free(chan
->lli_pool
, swdesc
->lli
[i
].hwdesc
, swdesc
->lli
[i
].hwdesc_addr
);
434 static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan
*chan
,
435 struct stm32_dma3_swdesc
*swdesc
)
439 for (i
= 0; i
< swdesc
->lli_size
; i
++)
440 dma_pool_free(chan
->lli_pool
, swdesc
->lli
[i
].hwdesc
, swdesc
->lli
[i
].hwdesc_addr
);
445 static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc
*vdesc
)
447 struct stm32_dma3_swdesc
*swdesc
= to_stm32_dma3_swdesc(vdesc
);
448 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(vdesc
->tx
.chan
);
450 stm32_dma3_chan_desc_free(chan
, swdesc
);
453 static void stm32_dma3_check_user_setting(struct stm32_dma3_chan
*chan
)
455 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
456 struct device
*dev
= chan2dev(chan
);
457 u32 ctr1
= readl_relaxed(ddata
->base
+ STM32_DMA3_CTR1(chan
->id
));
458 u32 cbr1
= readl_relaxed(ddata
->base
+ STM32_DMA3_CBR1(chan
->id
));
459 u32 csar
= readl_relaxed(ddata
->base
+ STM32_DMA3_CSAR(chan
->id
));
460 u32 cdar
= readl_relaxed(ddata
->base
+ STM32_DMA3_CDAR(chan
->id
));
461 u32 cllr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CLLR(chan
->id
));
462 u32 bndt
= FIELD_GET(CBR1_BNDT
, cbr1
);
463 u32 sdw
= 1 << FIELD_GET(CTR1_SDW_LOG2
, ctr1
);
464 u32 ddw
= 1 << FIELD_GET(CTR1_DDW_LOG2
, ctr1
);
465 u32 sap
= FIELD_GET(CTR1_SAP
, ctr1
);
466 u32 dap
= FIELD_GET(CTR1_DAP
, ctr1
);
468 if (!bndt
&& !FIELD_GET(CLLR_UB1
, cllr
))
469 dev_err(dev
, "null source block size and no update of this value\n");
471 dev_err(dev
, "source block size not multiple of src data width\n");
472 if (FIELD_GET(CTR1_PAM
, ctr1
) == CTR1_PAM_PACK_UNPACK
&& bndt
% ddw
)
473 dev_err(dev
, "(un)packing mode w/ src block size not multiple of dst data width\n");
475 dev_err(dev
, "unaligned source address not multiple of src data width\n");
477 dev_err(dev
, "unaligned destination address not multiple of dst data width\n");
478 if (sdw
== DMA_SLAVE_BUSWIDTH_8_BYTES
&& port_is_ahb(ddata
->ports_max_dw
[sap
]))
479 dev_err(dev
, "double-word source data width not supported on port %u\n", sap
);
480 if (ddw
== DMA_SLAVE_BUSWIDTH_8_BYTES
&& port_is_ahb(ddata
->ports_max_dw
[dap
]))
481 dev_err(dev
, "double-word destination data width not supported on port %u\n", dap
);
484 static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan
*chan
,
485 struct stm32_dma3_swdesc
*swdesc
,
486 u32 curr
, dma_addr_t src
, dma_addr_t dst
, u32 len
,
487 u32 ctr1
, u32 ctr2
, bool is_last
, bool is_cyclic
)
489 struct stm32_dma3_hwdesc
*hwdesc
;
493 hwdesc
= swdesc
->lli
[curr
].hwdesc
;
496 hwdesc
->cbr1
= FIELD_PREP(CBR1_BNDT
, len
);
502 next_lli
= swdesc
->lli
[0].hwdesc_addr
;
506 next_lli
= swdesc
->lli
[next
].hwdesc_addr
;
511 hwdesc
->cllr
|= CLLR_UT1
| CLLR_UT2
| CLLR_UB1
;
512 hwdesc
->cllr
|= CLLR_USA
| CLLR_UDA
| CLLR_ULL
;
513 hwdesc
->cllr
|= (next_lli
& CLLR_LA
);
517 * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read
518 * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel
519 * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors
520 * are reused (DMA_CTRL_REUSE).
526 static enum dma_slave_buswidth
stm32_dma3_get_max_dw(u32 chan_max_burst
,
527 enum stm32_dma3_port_data_width port_max_dw
,
528 u32 len
, dma_addr_t addr
)
530 enum dma_slave_buswidth max_dw
= get_chan_max_dw(port_max_dw
, chan_max_burst
);
532 /* len and addr must be a multiple of dw */
533 return 1 << __ffs(len
| addr
| max_dw
);
536 static u32
stm32_dma3_get_max_burst(u32 len
, enum dma_slave_buswidth dw
, u32 chan_max_burst
)
538 u32 max_burst
= chan_max_burst
? chan_max_burst
/ dw
: 1;
540 /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */
541 if (len
< chan_max_burst
)
542 max_burst
= len
/ dw
;
545 * HW doesn't modify the burst if burst size <= half of the fifo size.
546 * If len is not a multiple of burst size, last burst is shortened by HW.
551 static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan
*chan
, enum dma_transfer_direction dir
,
552 u32
*ccr
, u32
*ctr1
, u32
*ctr2
,
553 dma_addr_t src_addr
, dma_addr_t dst_addr
, u32 len
)
555 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
556 struct dma_device dma_device
= ddata
->dma_dev
;
557 u32 sdw
, ddw
, sbl_max
, dbl_max
, tcem
, init_dw
, init_bl_max
;
558 u32 _ctr1
= 0, _ctr2
= 0;
559 u32 ch_conf
= chan
->dt_config
.ch_conf
;
560 u32 tr_conf
= chan
->dt_config
.tr_conf
;
561 u32 sap
= FIELD_GET(STM32_DMA3_DT_SAP
, tr_conf
), sap_max_dw
;
562 u32 dap
= FIELD_GET(STM32_DMA3_DT_DAP
, tr_conf
), dap_max_dw
;
564 dev_dbg(chan2dev(chan
), "%s from %pad to %pad\n",
565 dmaengine_get_direction_text(dir
), &src_addr
, &dst_addr
);
567 sdw
= chan
->dma_config
.src_addr_width
? : get_chan_max_dw(sap
, chan
->max_burst
);
568 ddw
= chan
->dma_config
.dst_addr_width
? : get_chan_max_dw(dap
, chan
->max_burst
);
569 sbl_max
= chan
->dma_config
.src_maxburst
? : 1;
570 dbl_max
= chan
->dma_config
.dst_maxburst
? : 1;
572 /* Following conditions would raise User Setting Error interrupt */
573 if (!(dma_device
.src_addr_widths
& BIT(sdw
)) || !(dma_device
.dst_addr_widths
& BIT(ddw
))) {
574 dev_err(chan2dev(chan
), "Bus width (src=%u, dst=%u) not supported\n", sdw
, ddw
);
578 if (ddata
->ports_max_dw
[1] == DW_INVALID
&& (sap
|| dap
)) {
579 dev_err(chan2dev(chan
), "Only one master port, port 1 is not supported\n");
583 sap_max_dw
= ddata
->ports_max_dw
[sap
];
584 dap_max_dw
= ddata
->ports_max_dw
[dap
];
585 if ((port_is_ahb(sap_max_dw
) && sdw
== DMA_SLAVE_BUSWIDTH_8_BYTES
) ||
586 (port_is_ahb(dap_max_dw
) && ddw
== DMA_SLAVE_BUSWIDTH_8_BYTES
)) {
587 dev_err(chan2dev(chan
),
588 "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n",
593 if (FIELD_GET(STM32_DMA3_DT_SINC
, tr_conf
))
597 if (FIELD_GET(STM32_DMA3_DT_DINC
, tr_conf
))
602 _ctr2
|= FIELD_PREP(CTR2_REQSEL
, chan
->dt_config
.req_line
) & ~CTR2_SWREQ
;
603 if (FIELD_GET(STM32_DMA3_DT_BREQ
, tr_conf
))
605 if (dir
== DMA_DEV_TO_MEM
&& FIELD_GET(STM32_DMA3_DT_PFREQ
, tr_conf
))
607 tcem
= FIELD_GET(STM32_DMA3_DT_TCEM
, tr_conf
);
608 _ctr2
|= FIELD_PREP(CTR2_TCEM
, tcem
);
610 /* Store TCEM to know on which event TC flag occurred */
612 /* Store direction for residue computation */
613 chan
->dma_config
.direction
= dir
;
617 /* Set destination (device) data width and burst */
618 ddw
= min_t(u32
, ddw
, stm32_dma3_get_max_dw(chan
->max_burst
, dap_max_dw
,
620 dbl_max
= min_t(u32
, dbl_max
, stm32_dma3_get_max_burst(len
, ddw
, chan
->max_burst
));
622 /* Set source (memory) data width and burst */
623 sdw
= stm32_dma3_get_max_dw(chan
->max_burst
, sap_max_dw
, len
, src_addr
);
624 sbl_max
= stm32_dma3_get_max_burst(len
, sdw
, chan
->max_burst
);
626 _ctr1
|= FIELD_PREP(CTR1_SDW_LOG2
, ilog2(sdw
));
627 _ctr1
|= FIELD_PREP(CTR1_SBL_1
, sbl_max
- 1);
628 _ctr1
|= FIELD_PREP(CTR1_DDW_LOG2
, ilog2(ddw
));
629 _ctr1
|= FIELD_PREP(CTR1_DBL_1
, dbl_max
- 1);
632 _ctr1
|= FIELD_PREP(CTR1_PAM
, CTR1_PAM_PACK_UNPACK
);
633 /* Should never reach this case as ddw is clamped down */
634 if (len
& (ddw
- 1)) {
635 dev_err(chan2dev(chan
),
636 "Packing mode is enabled and len is not multiple of ddw");
647 /* Set source (device) data width and burst */
648 sdw
= min_t(u32
, sdw
, stm32_dma3_get_max_dw(chan
->max_burst
, sap_max_dw
,
650 sbl_max
= min_t(u32
, sbl_max
, stm32_dma3_get_max_burst(len
, sdw
, chan
->max_burst
));
652 /* Set destination (memory) data width and burst */
653 ddw
= stm32_dma3_get_max_dw(chan
->max_burst
, dap_max_dw
, len
, dst_addr
);
654 dbl_max
= stm32_dma3_get_max_burst(len
, ddw
, chan
->max_burst
);
656 _ctr1
|= FIELD_PREP(CTR1_SDW_LOG2
, ilog2(sdw
));
657 _ctr1
|= FIELD_PREP(CTR1_SBL_1
, sbl_max
- 1);
658 _ctr1
|= FIELD_PREP(CTR1_DDW_LOG2
, ilog2(ddw
));
659 _ctr1
|= FIELD_PREP(CTR1_DBL_1
, dbl_max
- 1);
662 _ctr1
|= FIELD_PREP(CTR1_PAM
, CTR1_PAM_PACK_UNPACK
);
663 /* Should never reach this case as ddw is clamped down */
664 if (len
& (ddw
- 1)) {
665 dev_err(chan2dev(chan
),
666 "Packing mode is enabled and len is not multiple of ddw\n");
677 /* Set source (memory) data width and burst */
679 init_bl_max
= sbl_max
;
680 sdw
= stm32_dma3_get_max_dw(chan
->max_burst
, sap_max_dw
, len
, src_addr
);
681 sbl_max
= stm32_dma3_get_max_burst(len
, sdw
, chan
->max_burst
);
682 if (chan
->config_set
& STM32_DMA3_CFG_SET_DMA
) {
683 sdw
= min_t(u32
, init_dw
, sdw
);
684 sbl_max
= min_t(u32
, init_bl_max
,
685 stm32_dma3_get_max_burst(len
, sdw
, chan
->max_burst
));
688 /* Set destination (memory) data width and burst */
690 init_bl_max
= dbl_max
;
691 ddw
= stm32_dma3_get_max_dw(chan
->max_burst
, dap_max_dw
, len
, dst_addr
);
692 dbl_max
= stm32_dma3_get_max_burst(len
, ddw
, chan
->max_burst
);
693 if (chan
->config_set
& STM32_DMA3_CFG_SET_DMA
) {
694 ddw
= min_t(u32
, init_dw
, ddw
);
695 dbl_max
= min_t(u32
, init_bl_max
,
696 stm32_dma3_get_max_burst(len
, ddw
, chan
->max_burst
));
699 _ctr1
|= FIELD_PREP(CTR1_SDW_LOG2
, ilog2(sdw
));
700 _ctr1
|= FIELD_PREP(CTR1_SBL_1
, sbl_max
- 1);
701 _ctr1
|= FIELD_PREP(CTR1_DDW_LOG2
, ilog2(ddw
));
702 _ctr1
|= FIELD_PREP(CTR1_DBL_1
, dbl_max
- 1);
705 _ctr1
|= FIELD_PREP(CTR1_PAM
, CTR1_PAM_PACK_UNPACK
);
706 /* Should never reach this case as ddw is clamped down */
707 if (len
& (ddw
- 1)) {
708 dev_err(chan2dev(chan
),
709 "Packing mode is enabled and len is not multiple of ddw");
714 /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */
720 dev_err(chan2dev(chan
), "Direction %s not supported\n",
721 dmaengine_get_direction_text(dir
));
725 *ccr
|= FIELD_PREP(CCR_PRIO
, FIELD_GET(STM32_DMA3_DT_PRIO
, ch_conf
));
729 dev_dbg(chan2dev(chan
), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",
730 __func__
, sdw
, sbl_max
, ddw
, dbl_max
);
735 static void stm32_dma3_chan_start(struct stm32_dma3_chan
*chan
)
737 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
738 struct virt_dma_desc
*vdesc
;
739 struct stm32_dma3_hwdesc
*hwdesc
;
743 vdesc
= vchan_next_desc(&chan
->vchan
);
748 list_del(&vdesc
->node
);
750 chan
->swdesc
= to_stm32_dma3_swdesc(vdesc
);
751 hwdesc
= chan
->swdesc
->lli
[0].hwdesc
;
753 stm32_dma3_chan_dump_hwdesc(chan
, chan
->swdesc
);
755 writel_relaxed(chan
->swdesc
->ccr
, ddata
->base
+ STM32_DMA3_CCR(id
));
756 writel_relaxed(hwdesc
->ctr1
, ddata
->base
+ STM32_DMA3_CTR1(id
));
757 writel_relaxed(hwdesc
->ctr2
, ddata
->base
+ STM32_DMA3_CTR2(id
));
758 writel_relaxed(hwdesc
->cbr1
, ddata
->base
+ STM32_DMA3_CBR1(id
));
759 writel_relaxed(hwdesc
->csar
, ddata
->base
+ STM32_DMA3_CSAR(id
));
760 writel_relaxed(hwdesc
->cdar
, ddata
->base
+ STM32_DMA3_CDAR(id
));
761 writel_relaxed(hwdesc
->cllr
, ddata
->base
+ STM32_DMA3_CLLR(id
));
763 /* Clear any pending interrupts */
764 csr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CSR(id
));
766 writel_relaxed(csr
, ddata
->base
+ STM32_DMA3_CFCR(id
));
768 stm32_dma3_chan_dump_reg(chan
);
770 ccr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CCR(id
));
771 writel_relaxed(ccr
| CCR_EN
, ddata
->base
+ STM32_DMA3_CCR(id
));
773 chan
->dma_status
= DMA_IN_PROGRESS
;
775 dev_dbg(chan2dev(chan
), "vchan %pK: started\n", &chan
->vchan
);
778 static int stm32_dma3_chan_suspend(struct stm32_dma3_chan
*chan
, bool susp
)
780 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
781 u32 csr
, ccr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CCR(chan
->id
)) & ~CCR_EN
;
789 writel_relaxed(ccr
, ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
792 ret
= readl_relaxed_poll_timeout_atomic(ddata
->base
+ STM32_DMA3_CSR(chan
->id
), csr
,
793 csr
& CSR_SUSPF
, 1, 10);
795 writel_relaxed(CFCR_SUSPF
, ddata
->base
+ STM32_DMA3_CFCR(chan
->id
));
797 stm32_dma3_chan_dump_reg(chan
);
803 static void stm32_dma3_chan_reset(struct stm32_dma3_chan
*chan
)
805 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
806 u32 ccr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CCR(chan
->id
)) & ~CCR_EN
;
808 writel_relaxed(ccr
|= CCR_RESET
, ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
811 static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc
*swdesc
, u32 cllr
, u32
*residue
)
813 u32 i
, lli_offset
, next_lli_offset
= cllr
& CLLR_LA
;
815 /* If cllr is null, it means it is either the last or single item */
817 return swdesc
->lli_size
- 1;
819 /* In cyclic mode, go fast and first check we are not on the last item */
820 if (swdesc
->cyclic
&& next_lli_offset
== (swdesc
->lli
[0].hwdesc_addr
& CLLR_LA
))
821 return swdesc
->lli_size
- 1;
823 /* As transfer is in progress, look backward from the last item */
824 for (i
= swdesc
->lli_size
- 1; i
> 0; i
--) {
825 *residue
+= FIELD_GET(CBR1_BNDT
, swdesc
->lli
[i
].hwdesc
->cbr1
);
826 lli_offset
= swdesc
->lli
[i
].hwdesc_addr
& CLLR_LA
;
827 if (lli_offset
== next_lli_offset
)
834 static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan
*chan
,
835 struct stm32_dma3_swdesc
*swdesc
,
836 struct dma_tx_state
*txstate
)
838 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
839 struct device
*dev
= chan2dev(chan
);
840 struct stm32_dma3_hwdesc
*hwdesc
;
841 u32 residue
, curr_lli
, csr
, cdar
, cbr1
, cllr
, bndt
, fifol
;
845 csr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CSR(chan
->id
));
846 if (!(csr
& CSR_IDLEF
) && chan
->dma_status
!= DMA_PAUSED
) {
847 /* Suspend current transfer to read registers for a snapshot */
848 writel_relaxed(swdesc
->ccr
| CCR_SUSP
, ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
849 ret
= readl_relaxed_poll_timeout_atomic(ddata
->base
+ STM32_DMA3_CSR(chan
->id
), csr
,
850 csr
& (CSR_SUSPF
| CSR_IDLEF
), 1, 10);
852 if (ret
|| ((csr
& CSR_TCF
) && (csr
& CSR_IDLEF
))) {
853 writel_relaxed(CFCR_SUSPF
, ddata
->base
+ STM32_DMA3_CFCR(chan
->id
));
854 writel_relaxed(swdesc
->ccr
, ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
856 dev_err(dev
, "Channel suspension timeout, csr=%08x\n", csr
);
860 /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */
861 if (!(csr
& CSR_IDLEF
))
862 dev_warn(dev
, "Can't get residue: channel still active, csr=%08x\n", csr
);
865 * If channel is not suspended, but Idle and Transfer Complete are set,
866 * linked-list is over, no residue
868 if (!(csr
& CSR_SUSPF
) && (csr
& CSR_TCF
) && (csr
& CSR_IDLEF
))
871 /* Read registers to have a snapshot */
872 cllr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CLLR(chan
->id
));
873 cbr1
= readl_relaxed(ddata
->base
+ STM32_DMA3_CBR1(chan
->id
));
874 cdar
= readl_relaxed(ddata
->base
+ STM32_DMA3_CDAR(chan
->id
));
876 /* Resume current transfer */
877 if (csr
& CSR_SUSPF
) {
878 writel_relaxed(CFCR_SUSPF
, ddata
->base
+ STM32_DMA3_CFCR(chan
->id
));
879 writel_relaxed(swdesc
->ccr
, ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
882 /* Add current BNDT */
883 bndt
= FIELD_GET(CBR1_BNDT
, cbr1
);
886 /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */
887 ret
= stm32_dma3_chan_get_curr_hwdesc(swdesc
, cllr
, &residue
);
889 dev_err(chan2dev(chan
), "Can't get residue: current hwdesc not found\n");
894 /* Read current FIFO level - in units of programmed destination data width */
895 hwdesc
= swdesc
->lli
[curr_lli
].hwdesc
;
896 fifol
= FIELD_GET(CSR_FIFOL
, csr
) * (1 << FIELD_GET(CTR1_DDW_LOG2
, hwdesc
->ctr1
));
897 /* If the FIFO contains as many bytes as its size, it can't contain more */
898 if (fifol
== (1 << (chan
->fifo_size
+ 1)))
899 goto skip_fifol_update
;
902 * In case of PACKING (Destination burst length > Source burst length) or UNPACKING
903 * (Source burst length > Destination burst length), bytes could be pending in the FIFO
904 * (to be packed up to Destination burst length or unpacked into Destination burst length
906 * BNDT is not reliable, as it reflects the number of bytes read from the source but not the
907 * number of bytes written to the destination.
908 * FIFOL is also not sufficient, because it reflects the number of available write beats in
909 * units of Destination data width but not the bytes not yet packed or unpacked.
910 * In case of Destination increment DINC, it is possible to compute the number of bytes in
912 * fifol_in_bytes = bytes_read - bytes_written.
914 pack_unpack
= !!(FIELD_GET(CTR1_PAM
, hwdesc
->ctr1
) == CTR1_PAM_PACK_UNPACK
);
915 if (pack_unpack
&& (hwdesc
->ctr1
& CTR1_DINC
)) {
916 int bytes_read
= FIELD_GET(CBR1_BNDT
, hwdesc
->cbr1
) - bndt
;
917 int bytes_written
= cdar
- hwdesc
->cdar
;
920 fifol
= bytes_read
- bytes_written
;
925 dev_dbg(chan2dev(chan
), "%u byte(s) in the FIFO\n", fifol
);
926 dma_set_in_flight_bytes(txstate
, fifol
);
928 * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from
929 * the source memory buffer, so just need to add fifol to residue in case of
930 * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination
933 if (chan
->dma_config
.direction
== DMA_DEV_TO_MEM
)
936 dma_set_residue(txstate
, residue
);
939 static int stm32_dma3_chan_stop(struct stm32_dma3_chan
*chan
)
941 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
945 chan
->dma_status
= DMA_COMPLETE
;
947 /* Disable interrupts */
948 ccr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
949 writel_relaxed(ccr
& ~(CCR_ALLIE
| CCR_EN
), ddata
->base
+ STM32_DMA3_CCR(chan
->id
));
951 if (!(ccr
& CCR_SUSP
) && (ccr
& CCR_EN
)) {
952 /* Suspend the channel */
953 ret
= stm32_dma3_chan_suspend(chan
, true);
955 dev_warn(chan2dev(chan
), "%s: timeout, data might be lost\n", __func__
);
959 * Reset the channel: this causes the reset of the FIFO and the reset of the channel
960 * internal state, the reset of CCR_EN and CCR_SUSP bits.
962 stm32_dma3_chan_reset(chan
);
967 static void stm32_dma3_chan_complete(struct stm32_dma3_chan
*chan
)
972 vchan_cookie_complete(&chan
->swdesc
->vdesc
);
974 stm32_dma3_chan_start(chan
);
977 static irqreturn_t
stm32_dma3_chan_irq(int irq
, void *devid
)
979 struct stm32_dma3_chan
*chan
= devid
;
980 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
983 spin_lock(&chan
->vchan
.lock
);
985 misr
= readl_relaxed(ddata
->base
+ STM32_DMA3_MISR
);
986 if (!(misr
& MISR_MIS(chan
->id
))) {
987 spin_unlock(&chan
->vchan
.lock
);
991 csr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CSR(chan
->id
));
992 ccr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CCR(chan
->id
)) & CCR_ALLIE
;
994 if (csr
& CSR_TCF
&& ccr
& CCR_TCIE
) {
995 if (chan
->swdesc
->cyclic
)
996 vchan_cyclic_callback(&chan
->swdesc
->vdesc
);
998 stm32_dma3_chan_complete(chan
);
1001 if (csr
& CSR_USEF
&& ccr
& CCR_USEIE
) {
1002 dev_err(chan2dev(chan
), "User setting error\n");
1003 chan
->dma_status
= DMA_ERROR
;
1004 /* CCR.EN automatically cleared by HW */
1005 stm32_dma3_check_user_setting(chan
);
1006 stm32_dma3_chan_reset(chan
);
1009 if (csr
& CSR_ULEF
&& ccr
& CCR_ULEIE
) {
1010 dev_err(chan2dev(chan
), "Update link transfer error\n");
1011 chan
->dma_status
= DMA_ERROR
;
1012 /* CCR.EN automatically cleared by HW */
1013 stm32_dma3_chan_reset(chan
);
1016 if (csr
& CSR_DTEF
&& ccr
& CCR_DTEIE
) {
1017 dev_err(chan2dev(chan
), "Data transfer error\n");
1018 chan
->dma_status
= DMA_ERROR
;
1019 /* CCR.EN automatically cleared by HW */
1020 stm32_dma3_chan_reset(chan
);
1024 * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set,
1025 * ensure HTF flag to be cleared, with other flags.
1027 csr
&= (ccr
| CCR_HTIE
);
1030 writel_relaxed(csr
, ddata
->base
+ STM32_DMA3_CFCR(chan
->id
));
1032 spin_unlock(&chan
->vchan
.lock
);
1037 static int stm32_dma3_alloc_chan_resources(struct dma_chan
*c
)
1039 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1040 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
1041 u32 id
= chan
->id
, csemcr
, ccid
;
1044 ret
= pm_runtime_resume_and_get(ddata
->dma_dev
.dev
);
1048 /* Ensure the channel is free */
1049 if (chan
->semaphore_mode
&&
1050 readl_relaxed(ddata
->base
+ STM32_DMA3_CSEMCR(chan
->id
)) & CSEMCR_SEM_MUTEX
) {
1055 chan
->lli_pool
= dmam_pool_create(dev_name(&c
->dev
->device
), c
->device
->dev
,
1056 sizeof(struct stm32_dma3_hwdesc
),
1057 __alignof__(struct stm32_dma3_hwdesc
), SZ_64K
);
1058 if (!chan
->lli_pool
) {
1059 dev_err(chan2dev(chan
), "Failed to create LLI pool\n");
1064 /* Take the channel semaphore */
1065 if (chan
->semaphore_mode
) {
1066 writel_relaxed(CSEMCR_SEM_MUTEX
, ddata
->base
+ STM32_DMA3_CSEMCR(id
));
1067 csemcr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CSEMCR(id
));
1068 ccid
= FIELD_GET(CSEMCR_SEM_CCID
, csemcr
);
1069 /* Check that the channel is well taken */
1070 if (ccid
!= CCIDCFGR_CID1
) {
1071 dev_err(chan2dev(chan
), "Not under CID1 control (in-use by CID%d)\n", ccid
);
1073 goto err_pool_destroy
;
1075 dev_dbg(chan2dev(chan
), "Under CID1 control (semcr=0x%08x)\n", csemcr
);
1081 dmam_pool_destroy(chan
->lli_pool
);
1082 chan
->lli_pool
= NULL
;
1085 pm_runtime_put_sync(ddata
->dma_dev
.dev
);
1090 static void stm32_dma3_free_chan_resources(struct dma_chan
*c
)
1092 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1093 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
1094 unsigned long flags
;
1096 /* Ensure channel is in idle state */
1097 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
1098 stm32_dma3_chan_stop(chan
);
1099 chan
->swdesc
= NULL
;
1100 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
1102 vchan_free_chan_resources(to_virt_chan(c
));
1104 dmam_pool_destroy(chan
->lli_pool
);
1105 chan
->lli_pool
= NULL
;
1107 /* Release the channel semaphore */
1108 if (chan
->semaphore_mode
)
1109 writel_relaxed(0, ddata
->base
+ STM32_DMA3_CSEMCR(chan
->id
));
1111 pm_runtime_put_sync(ddata
->dma_dev
.dev
);
1113 /* Reset configuration */
1114 memset(&chan
->dt_config
, 0, sizeof(chan
->dt_config
));
1115 memset(&chan
->dma_config
, 0, sizeof(chan
->dma_config
));
1116 chan
->config_set
= 0;
1119 static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan
*chan
,
1120 dma_addr_t dst
, dma_addr_t src
)
1122 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
1123 u32 dw
= get_chan_max_dw(ddata
->ports_max_dw
[0], chan
->max_burst
); /* port 0 by default */
1124 u32 burst
= chan
->max_burst
/ dw
;
1126 /* Initialize dt_config if channel not pre-configured through DT */
1127 if (!(chan
->config_set
& STM32_DMA3_CFG_SET_DT
)) {
1128 chan
->dt_config
.ch_conf
= FIELD_PREP(STM32_DMA3_DT_PRIO
, CCR_PRIO_VERY_HIGH
);
1129 chan
->dt_config
.ch_conf
|= FIELD_PREP(STM32_DMA3_DT_FIFO
, chan
->fifo_size
);
1130 chan
->dt_config
.tr_conf
= STM32_DMA3_DT_SINC
| STM32_DMA3_DT_DINC
;
1131 chan
->dt_config
.tr_conf
|= FIELD_PREP(STM32_DMA3_DT_TCEM
, CTR2_TCEM_CHANNEL
);
1134 /* Initialize dma_config if dmaengine_slave_config() not used */
1135 if (!(chan
->config_set
& STM32_DMA3_CFG_SET_DMA
)) {
1136 chan
->dma_config
.src_addr_width
= dw
;
1137 chan
->dma_config
.dst_addr_width
= dw
;
1138 chan
->dma_config
.src_maxburst
= burst
;
1139 chan
->dma_config
.dst_maxburst
= burst
;
1140 chan
->dma_config
.src_addr
= src
;
1141 chan
->dma_config
.dst_addr
= dst
;
1145 static struct dma_async_tx_descriptor
*stm32_dma3_prep_dma_memcpy(struct dma_chan
*c
,
1146 dma_addr_t dst
, dma_addr_t src
,
1147 size_t len
, unsigned long flags
)
1149 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1150 struct stm32_dma3_swdesc
*swdesc
;
1151 size_t next_size
, offset
;
1152 u32 count
, i
, ctr1
, ctr2
;
1154 count
= DIV_ROUND_UP(len
, STM32_DMA3_MAX_BLOCK_SIZE
);
1156 swdesc
= stm32_dma3_chan_desc_alloc(chan
, count
);
1160 if (chan
->config_set
!= STM32_DMA3_CFG_SET_BOTH
)
1161 stm32_dma3_init_chan_config_for_memcpy(chan
, dst
, src
);
1163 for (i
= 0, offset
= 0; offset
< len
; i
++, offset
+= next_size
) {
1167 remaining
= len
- offset
;
1168 next_size
= min_t(size_t, remaining
, STM32_DMA3_MAX_BLOCK_SIZE
);
1170 ret
= stm32_dma3_chan_prep_hw(chan
, DMA_MEM_TO_MEM
, &swdesc
->ccr
, &ctr1
, &ctr2
,
1171 src
+ offset
, dst
+ offset
, next_size
);
1175 stm32_dma3_chan_prep_hwdesc(chan
, swdesc
, i
, src
+ offset
, dst
+ offset
, next_size
,
1176 ctr1
, ctr2
, next_size
== remaining
, false);
1179 /* Enable Errors interrupts */
1180 swdesc
->ccr
|= CCR_USEIE
| CCR_ULEIE
| CCR_DTEIE
;
1181 /* Enable Transfer state interrupts */
1182 swdesc
->ccr
|= CCR_TCIE
;
1184 swdesc
->cyclic
= false;
1186 return vchan_tx_prep(&chan
->vchan
, &swdesc
->vdesc
, flags
);
1189 stm32_dma3_chan_desc_free(chan
, swdesc
);
1194 static struct dma_async_tx_descriptor
*stm32_dma3_prep_slave_sg(struct dma_chan
*c
,
1195 struct scatterlist
*sgl
,
1196 unsigned int sg_len
,
1197 enum dma_transfer_direction dir
,
1198 unsigned long flags
, void *context
)
1200 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1201 struct stm32_dma3_swdesc
*swdesc
;
1202 struct scatterlist
*sg
;
1204 dma_addr_t sg_addr
, dev_addr
, src
, dst
;
1205 u32 i
, j
, count
, ctr1
, ctr2
;
1209 for_each_sg(sgl
, sg
, sg_len
, i
) {
1210 len
= sg_dma_len(sg
);
1211 if (len
> STM32_DMA3_MAX_BLOCK_SIZE
)
1212 count
+= DIV_ROUND_UP(len
, STM32_DMA3_MAX_BLOCK_SIZE
) - 1;
1215 swdesc
= stm32_dma3_chan_desc_alloc(chan
, count
);
1219 /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */
1221 for_each_sg(sgl
, sg
, sg_len
, i
) {
1222 sg_addr
= sg_dma_address(sg
);
1223 dev_addr
= (dir
== DMA_MEM_TO_DEV
) ? chan
->dma_config
.dst_addr
:
1224 chan
->dma_config
.src_addr
;
1225 len
= sg_dma_len(sg
);
1228 size_t chunk
= min_t(size_t, len
, STM32_DMA3_MAX_BLOCK_SIZE
);
1230 if (dir
== DMA_MEM_TO_DEV
) {
1234 ret
= stm32_dma3_chan_prep_hw(chan
, dir
, &swdesc
->ccr
, &ctr1
, &ctr2
,
1237 if (FIELD_GET(CTR1_DINC
, ctr1
))
1239 } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */
1243 ret
= stm32_dma3_chan_prep_hw(chan
, dir
, &swdesc
->ccr
, &ctr1
, &ctr2
,
1246 if (FIELD_GET(CTR1_SINC
, ctr1
))
1253 stm32_dma3_chan_prep_hwdesc(chan
, swdesc
, j
, src
, dst
, chunk
,
1254 ctr1
, ctr2
, j
== (count
- 1), false);
1262 /* Enable Error interrupts */
1263 swdesc
->ccr
|= CCR_USEIE
| CCR_ULEIE
| CCR_DTEIE
;
1264 /* Enable Transfer state interrupts */
1265 swdesc
->ccr
|= CCR_TCIE
;
1267 swdesc
->cyclic
= false;
1269 return vchan_tx_prep(&chan
->vchan
, &swdesc
->vdesc
, flags
);
1272 stm32_dma3_chan_desc_free(chan
, swdesc
);
1277 static struct dma_async_tx_descriptor
*stm32_dma3_prep_dma_cyclic(struct dma_chan
*c
,
1278 dma_addr_t buf_addr
,
1279 size_t buf_len
, size_t period_len
,
1280 enum dma_transfer_direction dir
,
1281 unsigned long flags
)
1283 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1284 struct stm32_dma3_swdesc
*swdesc
;
1285 dma_addr_t src
, dst
;
1286 u32 count
, i
, ctr1
, ctr2
;
1289 if (!buf_len
|| !period_len
|| period_len
> STM32_DMA3_MAX_BLOCK_SIZE
) {
1290 dev_err(chan2dev(chan
), "Invalid buffer/period length\n");
1294 if (buf_len
% period_len
) {
1295 dev_err(chan2dev(chan
), "Buffer length not multiple of period length\n");
1299 count
= buf_len
/ period_len
;
1300 swdesc
= stm32_dma3_chan_desc_alloc(chan
, count
);
1304 if (dir
== DMA_MEM_TO_DEV
) {
1306 dst
= chan
->dma_config
.dst_addr
;
1308 ret
= stm32_dma3_chan_prep_hw(chan
, DMA_MEM_TO_DEV
, &swdesc
->ccr
, &ctr1
, &ctr2
,
1309 src
, dst
, period_len
);
1310 } else if (dir
== DMA_DEV_TO_MEM
) {
1311 src
= chan
->dma_config
.src_addr
;
1314 ret
= stm32_dma3_chan_prep_hw(chan
, DMA_DEV_TO_MEM
, &swdesc
->ccr
, &ctr1
, &ctr2
,
1315 src
, dst
, period_len
);
1317 dev_err(chan2dev(chan
), "Invalid direction\n");
1324 for (i
= 0; i
< count
; i
++) {
1325 if (dir
== DMA_MEM_TO_DEV
) {
1326 src
= buf_addr
+ i
* period_len
;
1327 dst
= chan
->dma_config
.dst_addr
;
1328 } else { /* (dir == DMA_DEV_TO_MEM) */
1329 src
= chan
->dma_config
.src_addr
;
1330 dst
= buf_addr
+ i
* period_len
;
1333 stm32_dma3_chan_prep_hwdesc(chan
, swdesc
, i
, src
, dst
, period_len
,
1334 ctr1
, ctr2
, i
== (count
- 1), true);
1337 /* Enable Error interrupts */
1338 swdesc
->ccr
|= CCR_USEIE
| CCR_ULEIE
| CCR_DTEIE
;
1339 /* Enable Transfer state interrupts */
1340 swdesc
->ccr
|= CCR_TCIE
;
1342 swdesc
->cyclic
= true;
1344 return vchan_tx_prep(&chan
->vchan
, &swdesc
->vdesc
, flags
);
1347 stm32_dma3_chan_desc_free(chan
, swdesc
);
1352 static void stm32_dma3_caps(struct dma_chan
*c
, struct dma_slave_caps
*caps
)
1354 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1356 if (!chan
->fifo_size
) {
1357 caps
->max_burst
= 0;
1358 caps
->src_addr_widths
&= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES
);
1359 caps
->dst_addr_widths
&= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES
);
1361 /* Burst transfer should not exceed half of the fifo size */
1362 caps
->max_burst
= chan
->max_burst
;
1363 if (caps
->max_burst
< DMA_SLAVE_BUSWIDTH_8_BYTES
) {
1364 caps
->src_addr_widths
&= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES
);
1365 caps
->dst_addr_widths
&= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES
);
1370 static int stm32_dma3_config(struct dma_chan
*c
, struct dma_slave_config
*config
)
1372 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1374 memcpy(&chan
->dma_config
, config
, sizeof(*config
));
1375 chan
->config_set
|= STM32_DMA3_CFG_SET_DMA
;
1380 static int stm32_dma3_pause(struct dma_chan
*c
)
1382 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1385 ret
= stm32_dma3_chan_suspend(chan
, true);
1389 chan
->dma_status
= DMA_PAUSED
;
1391 dev_dbg(chan2dev(chan
), "vchan %pK: paused\n", &chan
->vchan
);
1396 static int stm32_dma3_resume(struct dma_chan
*c
)
1398 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1400 stm32_dma3_chan_suspend(chan
, false);
1402 chan
->dma_status
= DMA_IN_PROGRESS
;
1404 dev_dbg(chan2dev(chan
), "vchan %pK: resumed\n", &chan
->vchan
);
1409 static int stm32_dma3_terminate_all(struct dma_chan
*c
)
1411 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1412 unsigned long flags
;
1415 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
1418 vchan_terminate_vdesc(&chan
->swdesc
->vdesc
);
1419 chan
->swdesc
= NULL
;
1422 stm32_dma3_chan_stop(chan
);
1424 vchan_get_all_descriptors(&chan
->vchan
, &head
);
1426 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
1427 vchan_dma_desc_free_list(&chan
->vchan
, &head
);
1429 dev_dbg(chan2dev(chan
), "vchan %pK: terminated\n", &chan
->vchan
);
1434 static void stm32_dma3_synchronize(struct dma_chan
*c
)
1436 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1438 vchan_synchronize(&chan
->vchan
);
1441 static enum dma_status
stm32_dma3_tx_status(struct dma_chan
*c
, dma_cookie_t cookie
,
1442 struct dma_tx_state
*txstate
)
1444 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1445 struct stm32_dma3_swdesc
*swdesc
= NULL
;
1446 enum dma_status status
;
1447 unsigned long flags
;
1448 struct virt_dma_desc
*vd
;
1450 status
= dma_cookie_status(c
, cookie
, txstate
);
1451 if (status
== DMA_COMPLETE
)
1455 return chan
->dma_status
;
1457 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
1459 vd
= vchan_find_desc(&chan
->vchan
, cookie
);
1461 swdesc
= to_stm32_dma3_swdesc(vd
);
1462 else if (chan
->swdesc
&& chan
->swdesc
->vdesc
.tx
.cookie
== cookie
)
1463 swdesc
= chan
->swdesc
;
1465 /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */
1467 stm32_dma3_chan_set_residue(chan
, swdesc
, txstate
);
1469 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
1471 return chan
->dma_status
;
1474 static void stm32_dma3_issue_pending(struct dma_chan
*c
)
1476 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1477 unsigned long flags
;
1479 spin_lock_irqsave(&chan
->vchan
.lock
, flags
);
1481 if (vchan_issue_pending(&chan
->vchan
) && !chan
->swdesc
) {
1482 dev_dbg(chan2dev(chan
), "vchan %pK: issued\n", &chan
->vchan
);
1483 stm32_dma3_chan_start(chan
);
1486 spin_unlock_irqrestore(&chan
->vchan
.lock
, flags
);
1489 static bool stm32_dma3_filter_fn(struct dma_chan
*c
, void *fn_param
)
1491 struct stm32_dma3_chan
*chan
= to_stm32_dma3_chan(c
);
1492 struct stm32_dma3_ddata
*ddata
= to_stm32_dma3_ddata(chan
);
1493 struct stm32_dma3_dt_conf
*conf
= fn_param
;
1497 dev_dbg(c
->device
->dev
, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n",
1498 __func__
, dma_chan_name(c
), conf
->req_line
, conf
->ch_conf
, conf
->tr_conf
);
1500 if (!of_property_read_u32(c
->device
->dev
->of_node
, "dma-channel-mask", &mask
))
1501 if (!(mask
& BIT(chan
->id
)))
1504 ret
= pm_runtime_resume_and_get(ddata
->dma_dev
.dev
);
1507 semcr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CSEMCR(chan
->id
));
1508 pm_runtime_put_sync(ddata
->dma_dev
.dev
);
1510 /* Check if chan is free */
1511 if (semcr
& CSEMCR_SEM_MUTEX
)
1514 /* Check if chan fifo fits well */
1515 if (FIELD_GET(STM32_DMA3_DT_FIFO
, conf
->ch_conf
) != chan
->fifo_size
)
1521 static struct dma_chan
*stm32_dma3_of_xlate(struct of_phandle_args
*dma_spec
, struct of_dma
*ofdma
)
1523 struct stm32_dma3_ddata
*ddata
= ofdma
->of_dma_data
;
1524 dma_cap_mask_t mask
= ddata
->dma_dev
.cap_mask
;
1525 struct stm32_dma3_dt_conf conf
;
1526 struct stm32_dma3_chan
*chan
;
1529 if (dma_spec
->args_count
< 3) {
1530 dev_err(ddata
->dma_dev
.dev
, "Invalid args count\n");
1534 conf
.req_line
= dma_spec
->args
[0];
1535 conf
.ch_conf
= dma_spec
->args
[1];
1536 conf
.tr_conf
= dma_spec
->args
[2];
1538 if (conf
.req_line
>= ddata
->dma_requests
) {
1539 dev_err(ddata
->dma_dev
.dev
, "Invalid request line\n");
1543 /* Request dma channel among the generic dma controller list */
1544 c
= dma_request_channel(mask
, stm32_dma3_filter_fn
, &conf
);
1546 dev_err(ddata
->dma_dev
.dev
, "No suitable channel found\n");
1550 chan
= to_stm32_dma3_chan(c
);
1551 chan
->dt_config
= conf
;
1552 chan
->config_set
|= STM32_DMA3_CFG_SET_DT
;
1557 static u32
stm32_dma3_check_rif(struct stm32_dma3_ddata
*ddata
)
1559 u32 chan_reserved
, mask
= 0, i
, ccidcfgr
, invalid_cid
= 0;
1561 /* Reserve Secure channels */
1562 chan_reserved
= readl_relaxed(ddata
->base
+ STM32_DMA3_SECCFGR
);
1565 * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of
1566 * the processor which is configuring and using the given channel.
1567 * In case CID filtering is not configured, dma-channel-mask property can be used to
1568 * specify available DMA channels to the kernel.
1570 of_property_read_u32(ddata
->dma_dev
.dev
->of_node
, "dma-channel-mask", &mask
);
1572 /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */
1573 for (i
= 0; i
< ddata
->dma_channels
; i
++) {
1574 ccidcfgr
= readl_relaxed(ddata
->base
+ STM32_DMA3_CCIDCFGR(i
));
1576 if (!(ccidcfgr
& CCIDCFGR_CFEN
)) { /* !CID-filtered */
1577 invalid_cid
|= BIT(i
);
1578 if (!(mask
& BIT(i
))) /* Not in dma-channel-mask */
1579 chan_reserved
|= BIT(i
);
1580 } else { /* CID-filtered */
1581 if (!(ccidcfgr
& CCIDCFGR_SEM_EN
)) { /* Static CID mode */
1582 if (FIELD_GET(CCIDCFGR_SCID
, ccidcfgr
) != CCIDCFGR_CID1
)
1583 chan_reserved
|= BIT(i
);
1584 } else { /* Semaphore mode */
1585 if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1
, ccidcfgr
))
1586 chan_reserved
|= BIT(i
);
1587 ddata
->chans
[i
].semaphore_mode
= true;
1590 dev_dbg(ddata
->dma_dev
.dev
, "chan%d: %s mode, %s\n", i
,
1591 !(ccidcfgr
& CCIDCFGR_CFEN
) ? "!CID-filtered" :
1592 ddata
->chans
[i
].semaphore_mode
? "Semaphore" : "Static CID",
1593 (chan_reserved
& BIT(i
)) ? "denied" :
1594 mask
& BIT(i
) ? "force allowed" : "allowed");
1598 dev_warn(ddata
->dma_dev
.dev
, "chan%*pbl have invalid CID configuration\n",
1599 ddata
->dma_channels
, &invalid_cid
);
1601 return chan_reserved
;
1604 static const struct of_device_id stm32_dma3_of_match
[] = {
1605 { .compatible
= "st,stm32mp25-dma3", },
1608 MODULE_DEVICE_TABLE(of
, stm32_dma3_of_match
);
1610 static int stm32_dma3_probe(struct platform_device
*pdev
)
1612 struct device_node
*np
= pdev
->dev
.of_node
;
1613 struct stm32_dma3_ddata
*ddata
;
1614 struct reset_control
*reset
;
1615 struct stm32_dma3_chan
*chan
;
1616 struct dma_device
*dma_dev
;
1617 u32 master_ports
, chan_reserved
, i
, verr
;
1621 ddata
= devm_kzalloc(&pdev
->dev
, sizeof(*ddata
), GFP_KERNEL
);
1624 platform_set_drvdata(pdev
, ddata
);
1626 dma_dev
= &ddata
->dma_dev
;
1628 ddata
->base
= devm_platform_ioremap_resource(pdev
, 0);
1629 if (IS_ERR(ddata
->base
))
1630 return PTR_ERR(ddata
->base
);
1632 ddata
->clk
= devm_clk_get(&pdev
->dev
, NULL
);
1633 if (IS_ERR(ddata
->clk
))
1634 return dev_err_probe(&pdev
->dev
, PTR_ERR(ddata
->clk
), "Failed to get clk\n");
1636 reset
= devm_reset_control_get_optional(&pdev
->dev
, NULL
);
1638 return dev_err_probe(&pdev
->dev
, PTR_ERR(reset
), "Failed to get reset\n");
1640 ret
= clk_prepare_enable(ddata
->clk
);
1642 return dev_err_probe(&pdev
->dev
, ret
, "Failed to enable clk\n");
1644 reset_control_reset(reset
);
1646 INIT_LIST_HEAD(&dma_dev
->channels
);
1648 dma_cap_set(DMA_SLAVE
, dma_dev
->cap_mask
);
1649 dma_cap_set(DMA_PRIVATE
, dma_dev
->cap_mask
);
1650 dma_cap_set(DMA_CYCLIC
, dma_dev
->cap_mask
);
1651 dma_cap_set(DMA_MEMCPY
, dma_dev
->cap_mask
);
1652 dma_dev
->dev
= &pdev
->dev
;
1654 * This controller supports up to 8-byte buswidth depending on the port used and the
1655 * channel, and can only access address at even boundaries, multiple of the buswidth.
1657 dma_dev
->copy_align
= DMAENGINE_ALIGN_8_BYTES
;
1658 dma_dev
->src_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE
) |
1659 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES
) |
1660 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
) |
1661 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES
);
1662 dma_dev
->dst_addr_widths
= BIT(DMA_SLAVE_BUSWIDTH_1_BYTE
) |
1663 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES
) |
1664 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES
) |
1665 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES
);
1666 dma_dev
->directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
) | BIT(DMA_MEM_TO_MEM
);
1668 dma_dev
->descriptor_reuse
= true;
1669 dma_dev
->max_sg_burst
= STM32_DMA3_MAX_SEG_SIZE
;
1670 dma_dev
->residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
1671 dma_dev
->device_alloc_chan_resources
= stm32_dma3_alloc_chan_resources
;
1672 dma_dev
->device_free_chan_resources
= stm32_dma3_free_chan_resources
;
1673 dma_dev
->device_prep_dma_memcpy
= stm32_dma3_prep_dma_memcpy
;
1674 dma_dev
->device_prep_slave_sg
= stm32_dma3_prep_slave_sg
;
1675 dma_dev
->device_prep_dma_cyclic
= stm32_dma3_prep_dma_cyclic
;
1676 dma_dev
->device_caps
= stm32_dma3_caps
;
1677 dma_dev
->device_config
= stm32_dma3_config
;
1678 dma_dev
->device_pause
= stm32_dma3_pause
;
1679 dma_dev
->device_resume
= stm32_dma3_resume
;
1680 dma_dev
->device_terminate_all
= stm32_dma3_terminate_all
;
1681 dma_dev
->device_synchronize
= stm32_dma3_synchronize
;
1682 dma_dev
->device_tx_status
= stm32_dma3_tx_status
;
1683 dma_dev
->device_issue_pending
= stm32_dma3_issue_pending
;
1685 /* if dma_channels is not modified, get it from hwcfgr1 */
1686 if (of_property_read_u32(np
, "dma-channels", &ddata
->dma_channels
)) {
1687 hwcfgr
= readl_relaxed(ddata
->base
+ STM32_DMA3_HWCFGR1
);
1688 ddata
->dma_channels
= FIELD_GET(G_NUM_CHANNELS
, hwcfgr
);
1691 /* if dma_requests is not modified, get it from hwcfgr2 */
1692 if (of_property_read_u32(np
, "dma-requests", &ddata
->dma_requests
)) {
1693 hwcfgr
= readl_relaxed(ddata
->base
+ STM32_DMA3_HWCFGR2
);
1694 ddata
->dma_requests
= FIELD_GET(G_MAX_REQ_ID
, hwcfgr
) + 1;
1697 /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */
1698 hwcfgr
= readl_relaxed(ddata
->base
+ STM32_DMA3_HWCFGR1
);
1699 master_ports
= FIELD_GET(G_MASTER_PORTS
, hwcfgr
);
1701 ddata
->ports_max_dw
[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC
, hwcfgr
);
1702 if (master_ports
== AXI64
|| master_ports
== AHB32
) /* Single master port */
1703 ddata
->ports_max_dw
[1] = DW_INVALID
;
1704 else /* Dual master ports */
1705 ddata
->ports_max_dw
[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC
, hwcfgr
);
1707 ddata
->chans
= devm_kcalloc(&pdev
->dev
, ddata
->dma_channels
, sizeof(*ddata
->chans
),
1709 if (!ddata
->chans
) {
1711 goto err_clk_disable
;
1714 chan_reserved
= stm32_dma3_check_rif(ddata
);
1716 if (chan_reserved
== GENMASK(ddata
->dma_channels
- 1, 0)) {
1718 dev_err_probe(&pdev
->dev
, ret
, "No channel available, abort registration\n");
1719 goto err_clk_disable
;
1722 /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */
1723 hwcfgr
= readl_relaxed(ddata
->base
+ STM32_DMA3_HWCFGR3
);
1724 hwcfgr
|= ((u64
)readl_relaxed(ddata
->base
+ STM32_DMA3_HWCFGR4
)) << 32;
1726 for (i
= 0; i
< ddata
->dma_channels
; i
++) {
1727 if (chan_reserved
& BIT(i
))
1730 chan
= &ddata
->chans
[i
];
1732 chan
->fifo_size
= get_chan_hwcfg(i
, G_FIFO_SIZE(i
), hwcfgr
);
1733 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */
1734 chan
->max_burst
= (chan
->fifo_size
) ? (1 << (chan
->fifo_size
+ 1)) / 2 : 0;
1737 ret
= dmaenginem_async_device_register(dma_dev
);
1739 goto err_clk_disable
;
1741 for (i
= 0; i
< ddata
->dma_channels
; i
++) {
1744 if (chan_reserved
& BIT(i
))
1747 chan
= &ddata
->chans
[i
];
1748 snprintf(name
, sizeof(name
), "dma%dchan%d", ddata
->dma_dev
.dev_id
, chan
->id
);
1750 chan
->vchan
.desc_free
= stm32_dma3_chan_vdesc_free
;
1751 vchan_init(&chan
->vchan
, dma_dev
);
1753 ret
= dma_async_device_channel_register(&ddata
->dma_dev
, &chan
->vchan
.chan
, name
);
1755 dev_err_probe(&pdev
->dev
, ret
, "Failed to register channel %s\n", name
);
1756 goto err_clk_disable
;
1759 ret
= platform_get_irq(pdev
, i
);
1761 goto err_clk_disable
;
1764 ret
= devm_request_irq(&pdev
->dev
, chan
->irq
, stm32_dma3_chan_irq
, 0,
1765 dev_name(chan2dev(chan
)), chan
);
1767 dev_err_probe(&pdev
->dev
, ret
, "Failed to request channel %s IRQ\n",
1768 dev_name(chan2dev(chan
)));
1769 goto err_clk_disable
;
1773 ret
= of_dma_controller_register(np
, stm32_dma3_of_xlate
, ddata
);
1775 dev_err_probe(&pdev
->dev
, ret
, "Failed to register controller\n");
1776 goto err_clk_disable
;
1779 verr
= readl_relaxed(ddata
->base
+ STM32_DMA3_VERR
);
1781 pm_runtime_set_active(&pdev
->dev
);
1782 pm_runtime_enable(&pdev
->dev
);
1783 pm_runtime_get_noresume(&pdev
->dev
);
1784 pm_runtime_put(&pdev
->dev
);
1786 dev_info(&pdev
->dev
, "STM32 DMA3 registered rev:%lu.%lu\n",
1787 FIELD_GET(VERR_MAJREV
, verr
), FIELD_GET(VERR_MINREV
, verr
));
1792 clk_disable_unprepare(ddata
->clk
);
1797 static void stm32_dma3_remove(struct platform_device
*pdev
)
1799 pm_runtime_disable(&pdev
->dev
);
1802 static int stm32_dma3_runtime_suspend(struct device
*dev
)
1804 struct stm32_dma3_ddata
*ddata
= dev_get_drvdata(dev
);
1806 clk_disable_unprepare(ddata
->clk
);
1811 static int stm32_dma3_runtime_resume(struct device
*dev
)
1813 struct stm32_dma3_ddata
*ddata
= dev_get_drvdata(dev
);
1816 ret
= clk_prepare_enable(ddata
->clk
);
1818 dev_err(dev
, "Failed to enable clk: %d\n", ret
);
1823 static const struct dev_pm_ops stm32_dma3_pm_ops
= {
1824 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend
, pm_runtime_force_resume
)
1825 RUNTIME_PM_OPS(stm32_dma3_runtime_suspend
, stm32_dma3_runtime_resume
, NULL
)
1828 static struct platform_driver stm32_dma3_driver
= {
1829 .probe
= stm32_dma3_probe
,
1830 .remove_new
= stm32_dma3_remove
,
1832 .name
= "stm32-dma3",
1833 .of_match_table
= stm32_dma3_of_match
,
1834 .pm
= pm_ptr(&stm32_dma3_pm_ops
),
1838 static int __init
stm32_dma3_init(void)
1840 return platform_driver_register(&stm32_dma3_driver
);
1843 subsys_initcall(stm32_dma3_init
);
1845 MODULE_DESCRIPTION("STM32 DMA3 controller driver");
1846 MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>");
1847 MODULE_LICENSE("GPL");