?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / drivers / dma / stm32 / stm32-dma3.c
blob0c6c4258b19561c94f1c68f26ade16b82660ebe6
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * STM32 DMA3 controller driver
4 *
5 * Copyright (C) STMicroelectronics 2024
6 * Author(s): Amelie Delaunay <amelie.delaunay@foss.st.com>
7 */
8
9 #include <linux/bitfield.h>
10 #include <linux/clk.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmaengine.h>
13 #include <linux/dmapool.h>
14 #include <linux/init.h>
15 #include <linux/iopoll.h>
16 #include <linux/list.h>
17 #include <linux/module.h>
18 #include <linux/of_dma.h>
19 #include <linux/platform_device.h>
20 #include <linux/pm_runtime.h>
21 #include <linux/reset.h>
22 #include <linux/slab.h>
23
24 #include "../virt-dma.h"
25
26 #define STM32_DMA3_SECCFGR 0x00
27 #define STM32_DMA3_PRIVCFGR 0x04
28 #define STM32_DMA3_RCFGLOCKR 0x08
29 #define STM32_DMA3_MISR 0x0c
30 #define STM32_DMA3_SMISR 0x10
31
32 #define STM32_DMA3_CLBAR(x) (0x50 + 0x80 * (x))
33 #define STM32_DMA3_CCIDCFGR(x) (0x54 + 0x80 * (x))
34 #define STM32_DMA3_CSEMCR(x) (0x58 + 0x80 * (x))
35 #define STM32_DMA3_CFCR(x) (0x5c + 0x80 * (x))
36 #define STM32_DMA3_CSR(x) (0x60 + 0x80 * (x))
37 #define STM32_DMA3_CCR(x) (0x64 + 0x80 * (x))
38 #define STM32_DMA3_CTR1(x) (0x90 + 0x80 * (x))
39 #define STM32_DMA3_CTR2(x) (0x94 + 0x80 * (x))
40 #define STM32_DMA3_CBR1(x) (0x98 + 0x80 * (x))
41 #define STM32_DMA3_CSAR(x) (0x9c + 0x80 * (x))
42 #define STM32_DMA3_CDAR(x) (0xa0 + 0x80 * (x))
43 #define STM32_DMA3_CLLR(x) (0xcc + 0x80 * (x))
44
45 #define STM32_DMA3_HWCFGR13 0xfc0 /* G_PER_CTRL(X) x=8..15 */
46 #define STM32_DMA3_HWCFGR12 0xfc4 /* G_PER_CTRL(X) x=0..7 */
47 #define STM32_DMA3_HWCFGR4 0xfe4 /* G_FIFO_SIZE(X) x=8..15 */
48 #define STM32_DMA3_HWCFGR3 0xfe8 /* G_FIFO_SIZE(X) x=0..7 */
49 #define STM32_DMA3_HWCFGR2 0xfec /* G_MAX_REQ_ID */
50 #define STM32_DMA3_HWCFGR1 0xff0 /* G_MASTER_PORTS, G_NUM_CHANNELS, G_Mx_DATA_WIDTH */
51 #define STM32_DMA3_VERR 0xff4
52
53 /* SECCFGR DMA secure configuration register */
54 #define SECCFGR_SEC(x) BIT(x)
55
56 /* MISR DMA non-secure/secure masked interrupt status register */
57 #define MISR_MIS(x) BIT(x)
58
59 /* CxLBAR DMA channel x linked_list base address register */
60 #define CLBAR_LBA GENMASK(31, 16)
61
62 /* CxCIDCFGR DMA channel x CID register */
63 #define CCIDCFGR_CFEN BIT(0)
64 #define CCIDCFGR_SEM_EN BIT(1)
65 #define CCIDCFGR_SCID GENMASK(5, 4)
66 #define CCIDCFGR_SEM_WLIST_CID0 BIT(16)
67 #define CCIDCFGR_SEM_WLIST_CID1 BIT(17)
68 #define CCIDCFGR_SEM_WLIST_CID2 BIT(18)
69
70 enum ccidcfgr_cid {
71 CCIDCFGR_CID0,
72 CCIDCFGR_CID1,
73 CCIDCFGR_CID2,
74 };
75
76 /* CxSEMCR DMA channel x semaphore control register */
77 #define CSEMCR_SEM_MUTEX BIT(0)
78 #define CSEMCR_SEM_CCID GENMASK(5, 4)
79
80 /* CxFCR DMA channel x flag clear register */
81 #define CFCR_TCF BIT(8)
82 #define CFCR_HTF BIT(9)
83 #define CFCR_DTEF BIT(10)
84 #define CFCR_ULEF BIT(11)
85 #define CFCR_USEF BIT(12)
86 #define CFCR_SUSPF BIT(13)
87
88 /* CxSR DMA channel x status register */
89 #define CSR_IDLEF BIT(0)
90 #define CSR_TCF BIT(8)
91 #define CSR_HTF BIT(9)
92 #define CSR_DTEF BIT(10)
93 #define CSR_ULEF BIT(11)
94 #define CSR_USEF BIT(12)
95 #define CSR_SUSPF BIT(13)
96 #define CSR_ALL_F GENMASK(13, 8)
97 #define CSR_FIFOL GENMASK(24, 16)
98
99 /* CxCR DMA channel x control register */
100 #define CCR_EN BIT(0)
101 #define CCR_RESET BIT(1)
102 #define CCR_SUSP BIT(2)
103 #define CCR_TCIE BIT(8)
104 #define CCR_HTIE BIT(9)
105 #define CCR_DTEIE BIT(10)
106 #define CCR_ULEIE BIT(11)
107 #define CCR_USEIE BIT(12)
108 #define CCR_SUSPIE BIT(13)
109 #define CCR_ALLIE GENMASK(13, 8)
110 #define CCR_LSM BIT(16)
111 #define CCR_LAP BIT(17)
112 #define CCR_PRIO GENMASK(23, 22)
113
114 enum ccr_prio {
115 CCR_PRIO_LOW,
116 CCR_PRIO_MID,
117 CCR_PRIO_HIGH,
118 CCR_PRIO_VERY_HIGH,
119 };
120
121 /* CxTR1 DMA channel x transfer register 1 */
122 #define CTR1_SINC BIT(3)
123 #define CTR1_SBL_1 GENMASK(9, 4)
124 #define CTR1_DINC BIT(19)
125 #define CTR1_DBL_1 GENMASK(25, 20)
126 #define CTR1_SDW_LOG2 GENMASK(1, 0)
127 #define CTR1_PAM GENMASK(12, 11)
128 #define CTR1_SAP BIT(14)
129 #define CTR1_DDW_LOG2 GENMASK(17, 16)
130 #define CTR1_DAP BIT(30)
131
132 enum ctr1_dw {
133 CTR1_DW_BYTE,
134 CTR1_DW_HWORD,
135 CTR1_DW_WORD,
136 CTR1_DW_DWORD, /* Depends on HWCFGR1.G_M0_DATA_WIDTH_ENC and .G_M1_DATA_WIDTH_ENC */
137 };
138
139 enum ctr1_pam {
140 CTR1_PAM_0S_LT, /* if DDW > SDW, padded with 0s else left-truncated */
141 CTR1_PAM_SE_RT, /* if DDW > SDW, sign extended else right-truncated */
142 CTR1_PAM_PACK_UNPACK, /* FIFO queued */
143 };
144
145 /* CxTR2 DMA channel x transfer register 2 */
146 #define CTR2_REQSEL GENMASK(7, 0)
147 #define CTR2_SWREQ BIT(9)
148 #define CTR2_DREQ BIT(10)
149 #define CTR2_BREQ BIT(11)
150 #define CTR2_PFREQ BIT(12)
151 #define CTR2_TCEM GENMASK(31, 30)
152
153 enum ctr2_tcem {
154 CTR2_TCEM_BLOCK,
155 CTR2_TCEM_REPEAT_BLOCK,
156 CTR2_TCEM_LLI,
157 CTR2_TCEM_CHANNEL,
158 };
159
160 /* CxBR1 DMA channel x block register 1 */
161 #define CBR1_BNDT GENMASK(15, 0)
162
163 /* CxLLR DMA channel x linked-list address register */
164 #define CLLR_LA GENMASK(15, 2)
165 #define CLLR_ULL BIT(16)
166 #define CLLR_UDA BIT(27)
167 #define CLLR_USA BIT(28)
168 #define CLLR_UB1 BIT(29)
169 #define CLLR_UT2 BIT(30)
170 #define CLLR_UT1 BIT(31)
171
172 /* HWCFGR13 DMA hardware configuration register 13 x=8..15 */
173 /* HWCFGR12 DMA hardware configuration register 12 x=0..7 */
174 #define G_PER_CTRL(x) (ULL(0x1) << (4 * (x)))
175
176 /* HWCFGR4 DMA hardware configuration register 4 x=8..15 */
177 /* HWCFGR3 DMA hardware configuration register 3 x=0..7 */
178 #define G_FIFO_SIZE(x) (ULL(0x7) << (4 * (x)))
179
180 #define get_chan_hwcfg(x, mask, reg) (((reg) & (mask)) >> (4 * (x)))
181
182 /* HWCFGR2 DMA hardware configuration register 2 */
183 #define G_MAX_REQ_ID GENMASK(7, 0)
184
185 /* HWCFGR1 DMA hardware configuration register 1 */
186 #define G_MASTER_PORTS GENMASK(2, 0)
187 #define G_NUM_CHANNELS GENMASK(12, 8)
188 #define G_M0_DATA_WIDTH_ENC GENMASK(25, 24)
189 #define G_M1_DATA_WIDTH_ENC GENMASK(29, 28)
190
191 enum stm32_dma3_master_ports {
192 AXI64, /* 1x AXI: 64-bit port 0 */
193 AHB32, /* 1x AHB: 32-bit port 0 */
194 AHB32_AHB32, /* 2x AHB: 32-bit port 0 and 32-bit port 1 */
195 AXI64_AHB32, /* 1x AXI 64-bit port 0 and 1x AHB 32-bit port 1 */
196 AXI64_AXI64, /* 2x AXI: 64-bit port 0 and 64-bit port 1 */
197 AXI128_AHB32, /* 1x AXI 128-bit port 0 and 1x AHB 32-bit port 1 */
198 };
199
200 enum stm32_dma3_port_data_width {
201 DW_32, /* 32-bit, for AHB */
202 DW_64, /* 64-bit, for AXI */
203 DW_128, /* 128-bit, for AXI */
204 DW_INVALID,
205 };
206
207 /* VERR DMA version register */
208 #define VERR_MINREV GENMASK(3, 0)
209 #define VERR_MAJREV GENMASK(7, 4)
210
211 /* Device tree */
212 /* struct stm32_dma3_dt_conf */
213 /* .ch_conf */
214 #define STM32_DMA3_DT_PRIO GENMASK(1, 0) /* CCR_PRIO */
215 #define STM32_DMA3_DT_FIFO GENMASK(7, 4)
216 /* .tr_conf */
217 #define STM32_DMA3_DT_SINC BIT(0) /* CTR1_SINC */
218 #define STM32_DMA3_DT_SAP BIT(1) /* CTR1_SAP */
219 #define STM32_DMA3_DT_DINC BIT(4) /* CTR1_DINC */
220 #define STM32_DMA3_DT_DAP BIT(5) /* CTR1_DAP */
221 #define STM32_DMA3_DT_BREQ BIT(8) /* CTR2_BREQ */
222 #define STM32_DMA3_DT_PFREQ BIT(9) /* CTR2_PFREQ */
223 #define STM32_DMA3_DT_TCEM GENMASK(13, 12) /* CTR2_TCEM */
224 #define STM32_DMA3_DT_NOPACK BIT(16) /* CTR1_PAM */
225 #define STM32_DMA3_DT_NOREFACT BIT(17)
226
227 /* struct stm32_dma3_chan .config_set bitfield */
228 #define STM32_DMA3_CFG_SET_DT BIT(0)
229 #define STM32_DMA3_CFG_SET_DMA BIT(1)
230 #define STM32_DMA3_CFG_SET_BOTH (STM32_DMA3_CFG_SET_DT | STM32_DMA3_CFG_SET_DMA)
231
232 #define STM32_DMA3_MAX_BLOCK_SIZE ALIGN_DOWN(CBR1_BNDT, 64)
233 #define STM32_DMA3_MAX_BURST_LEN (1 + min_t(u32, FIELD_MAX(CTR1_SBL_1), \
234 FIELD_MAX(CTR1_DBL_1)))
235 #define port_is_ahb(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \
236 ((_maxdw) != DW_INVALID) && ((_maxdw) == DW_32); })
237 #define port_is_axi(maxdw) ({ typeof(maxdw) (_maxdw) = (maxdw); \
238 ((_maxdw) != DW_INVALID) && ((_maxdw) != DW_32); })
239 #define get_chan_max_dw(maxdw, maxburst)((port_is_ahb(maxdw) || \
240 (maxburst) < DMA_SLAVE_BUSWIDTH_8_BYTES) ? \
241 DMA_SLAVE_BUSWIDTH_4_BYTES : DMA_SLAVE_BUSWIDTH_8_BYTES)
242
243 /* Static linked-list data structure (depends on update bits UT1/UT2/UB1/USA/UDA/ULL) */
244 struct stm32_dma3_hwdesc {
245 u32 ctr1;
246 u32 ctr2;
247 u32 cbr1;
248 u32 csar;
249 u32 cdar;
250 u32 cllr;
251 } __packed __aligned(32);
252
253 /*
254 * CLLR_LA / sizeof(struct stm32_dma3_hwdesc) represents the number of hdwdesc that can be addressed
255 * by the pointer to the next linked-list data structure. The __aligned forces the 32-byte
256 * alignment. So use hardcoded 32. Multiplied by the max block size of each item, it represents
257 * the sg size limitation.
258 */
259 #define STM32_DMA3_MAX_SEG_SIZE ((CLLR_LA / 32) * STM32_DMA3_MAX_BLOCK_SIZE)
260
261 /*
262 * Linked-list items
263 */
264 struct stm32_dma3_lli {
265 struct stm32_dma3_hwdesc *hwdesc;
266 dma_addr_t hwdesc_addr;
267 };
268
269 struct stm32_dma3_swdesc {
270 struct virt_dma_desc vdesc;
271 u32 ccr;
272 bool cyclic;
273 u32 lli_size;
274 struct stm32_dma3_lli lli[] __counted_by(lli_size);
275 };
276
277 struct stm32_dma3_dt_conf {
278 u32 ch_id;
279 u32 req_line;
280 u32 ch_conf;
281 u32 tr_conf;
282 };
283
284 struct stm32_dma3_chan {
285 struct virt_dma_chan vchan;
286 u32 id;
287 int irq;
288 u32 fifo_size;
289 u32 max_burst;
290 bool semaphore_mode;
291 struct stm32_dma3_dt_conf dt_config;
292 struct dma_slave_config dma_config;
293 u8 config_set;
294 struct dma_pool *lli_pool;
295 struct stm32_dma3_swdesc *swdesc;
296 enum ctr2_tcem tcem;
297 u32 dma_status;
298 };
299
300 struct stm32_dma3_pdata {
301 u32 axi_max_burst_len;
302 };
303
304 struct stm32_dma3_ddata {
305 struct dma_device dma_dev;
306 void __iomem *base;
307 struct clk *clk;
308 struct stm32_dma3_chan *chans;
309 u32 dma_channels;
310 u32 dma_requests;
311 enum stm32_dma3_port_data_width ports_max_dw[2];
312 u32 axi_max_burst_len;
313 };
314
315 static inline struct stm32_dma3_ddata *to_stm32_dma3_ddata(struct stm32_dma3_chan *chan)
316 {
317 return container_of(chan->vchan.chan.device, struct stm32_dma3_ddata, dma_dev);
318 }
319
320 static inline struct stm32_dma3_chan *to_stm32_dma3_chan(struct dma_chan *c)
321 {
322 return container_of(c, struct stm32_dma3_chan, vchan.chan);
323 }
324
325 static inline struct stm32_dma3_swdesc *to_stm32_dma3_swdesc(struct virt_dma_desc *vdesc)
326 {
327 return container_of(vdesc, struct stm32_dma3_swdesc, vdesc);
328 }
329
330 static struct device *chan2dev(struct stm32_dma3_chan *chan)
331 {
332 return &chan->vchan.chan.dev->device;
333 }
334
335 static void stm32_dma3_chan_dump_reg(struct stm32_dma3_chan *chan)
336 {
337 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
338 struct device *dev = chan2dev(chan);
339 u32 id = chan->id, offset;
340
341 offset = STM32_DMA3_SECCFGR;
342 dev_dbg(dev, "SECCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));
343 offset = STM32_DMA3_PRIVCFGR;
344 dev_dbg(dev, "PRIVCFGR(0x%03x): %08x\n", offset, readl_relaxed(ddata->base + offset));
345 offset = STM32_DMA3_CCIDCFGR(id);
346 dev_dbg(dev, "C%dCIDCFGR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
347 offset = STM32_DMA3_CSEMCR(id);
348 dev_dbg(dev, "C%dSEMCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
349 offset = STM32_DMA3_CSR(id);
350 dev_dbg(dev, "C%dSR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
351 offset = STM32_DMA3_CCR(id);
352 dev_dbg(dev, "C%dCR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
353 offset = STM32_DMA3_CTR1(id);
354 dev_dbg(dev, "C%dTR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
355 offset = STM32_DMA3_CTR2(id);
356 dev_dbg(dev, "C%dTR2(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
357 offset = STM32_DMA3_CBR1(id);
358 dev_dbg(dev, "C%dBR1(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
359 offset = STM32_DMA3_CSAR(id);
360 dev_dbg(dev, "C%dSAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
361 offset = STM32_DMA3_CDAR(id);
362 dev_dbg(dev, "C%dDAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
363 offset = STM32_DMA3_CLLR(id);
364 dev_dbg(dev, "C%dLLR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
365 offset = STM32_DMA3_CLBAR(id);
366 dev_dbg(dev, "C%dLBAR(0x%03x): %08x\n", id, offset, readl_relaxed(ddata->base + offset));
367 }
368
369 static void stm32_dma3_chan_dump_hwdesc(struct stm32_dma3_chan *chan,
370 struct stm32_dma3_swdesc *swdesc)
371 {
372 struct stm32_dma3_hwdesc *hwdesc;
373 int i;
374
375 for (i = 0; i < swdesc->lli_size; i++) {
376 hwdesc = swdesc->lli[i].hwdesc;
377 if (i)
378 dev_dbg(chan2dev(chan), "V\n");
379 dev_dbg(chan2dev(chan), "[%d]@%pad\n", i, &swdesc->lli[i].hwdesc_addr);
380 dev_dbg(chan2dev(chan), "| C%dTR1: %08x\n", chan->id, hwdesc->ctr1);
381 dev_dbg(chan2dev(chan), "| C%dTR2: %08x\n", chan->id, hwdesc->ctr2);
382 dev_dbg(chan2dev(chan), "| C%dBR1: %08x\n", chan->id, hwdesc->cbr1);
383 dev_dbg(chan2dev(chan), "| C%dSAR: %08x\n", chan->id, hwdesc->csar);
384 dev_dbg(chan2dev(chan), "| C%dDAR: %08x\n", chan->id, hwdesc->cdar);
385 dev_dbg(chan2dev(chan), "| C%dLLR: %08x\n", chan->id, hwdesc->cllr);
386 }
387
388 if (swdesc->cyclic) {
389 dev_dbg(chan2dev(chan), "|\n");
390 dev_dbg(chan2dev(chan), "-->[0]@%pad\n", &swdesc->lli[0].hwdesc_addr);
391 } else {
392 dev_dbg(chan2dev(chan), "X\n");
393 }
394 }
395
396 static struct stm32_dma3_swdesc *stm32_dma3_chan_desc_alloc(struct stm32_dma3_chan *chan, u32 count)
397 {
398 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
399 struct stm32_dma3_swdesc *swdesc;
400 int i;
401
402 /*
403 * If the memory to be allocated for the number of hwdesc (6 u32 members but 32-bytes
404 * aligned) is greater than the maximum address of CLLR_LA, then the last items can't be
405 * addressed, so abort the allocation.
406 */
407 if ((count * 32) > CLLR_LA) {
408 dev_err(chan2dev(chan), "Transfer is too big (> %luB)\n", STM32_DMA3_MAX_SEG_SIZE);
409 return NULL;
410 }
411
412 swdesc = kzalloc(struct_size(swdesc, lli, count), GFP_NOWAIT);
413 if (!swdesc)
414 return NULL;
415 swdesc->lli_size = count;
416
417 for (i = 0; i < count; i++) {
418 swdesc->lli[i].hwdesc = dma_pool_zalloc(chan->lli_pool, GFP_NOWAIT,
419 &swdesc->lli[i].hwdesc_addr);
420 if (!swdesc->lli[i].hwdesc)
421 goto err_pool_free;
422 }
423 swdesc->ccr = 0;
424
425 /* Set LL base address */
426 writel_relaxed(swdesc->lli[0].hwdesc_addr & CLBAR_LBA,
427 ddata->base + STM32_DMA3_CLBAR(chan->id));
428
429 /* Set LL allocated port */
430 swdesc->ccr &= ~CCR_LAP;
431
432 return swdesc;
433
434 err_pool_free:
435 dev_err(chan2dev(chan), "Failed to alloc descriptors\n");
436 while (--i >= 0)
437 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
438 kfree(swdesc);
439
440 return NULL;
441 }
442
443 static void stm32_dma3_chan_desc_free(struct stm32_dma3_chan *chan,
444 struct stm32_dma3_swdesc *swdesc)
445 {
446 int i;
447
448 for (i = 0; i < swdesc->lli_size; i++)
449 dma_pool_free(chan->lli_pool, swdesc->lli[i].hwdesc, swdesc->lli[i].hwdesc_addr);
450
451 kfree(swdesc);
452 }
453
454 static void stm32_dma3_chan_vdesc_free(struct virt_dma_desc *vdesc)
455 {
456 struct stm32_dma3_swdesc *swdesc = to_stm32_dma3_swdesc(vdesc);
457 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(vdesc->tx.chan);
458
459 stm32_dma3_chan_desc_free(chan, swdesc);
460 }
461
462 static void stm32_dma3_check_user_setting(struct stm32_dma3_chan *chan)
463 {
464 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
465 struct device *dev = chan2dev(chan);
466 u32 ctr1 = readl_relaxed(ddata->base + STM32_DMA3_CTR1(chan->id));
467 u32 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
468 u32 csar = readl_relaxed(ddata->base + STM32_DMA3_CSAR(chan->id));
469 u32 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));
470 u32 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));
471 u32 bndt = FIELD_GET(CBR1_BNDT, cbr1);
472 u32 sdw = 1 << FIELD_GET(CTR1_SDW_LOG2, ctr1);
473 u32 ddw = 1 << FIELD_GET(CTR1_DDW_LOG2, ctr1);
474 u32 sap = FIELD_GET(CTR1_SAP, ctr1);
475 u32 dap = FIELD_GET(CTR1_DAP, ctr1);
476
477 if (!bndt && !FIELD_GET(CLLR_UB1, cllr))
478 dev_err(dev, "null source block size and no update of this value\n");
479 if (bndt % sdw)
480 dev_err(dev, "source block size not multiple of src data width\n");
481 if (FIELD_GET(CTR1_PAM, ctr1) == CTR1_PAM_PACK_UNPACK && bndt % ddw)
482 dev_err(dev, "(un)packing mode w/ src block size not multiple of dst data width\n");
483 if (csar % sdw)
484 dev_err(dev, "unaligned source address not multiple of src data width\n");
485 if (cdar % ddw)
486 dev_err(dev, "unaligned destination address not multiple of dst data width\n");
487 if (sdw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[sap]))
488 dev_err(dev, "double-word source data width not supported on port %u\n", sap);
489 if (ddw == DMA_SLAVE_BUSWIDTH_8_BYTES && port_is_ahb(ddata->ports_max_dw[dap]))
490 dev_err(dev, "double-word destination data width not supported on port %u\n", dap);
491 }
492
493 static void stm32_dma3_chan_prep_hwdesc(struct stm32_dma3_chan *chan,
494 struct stm32_dma3_swdesc *swdesc,
495 u32 curr, dma_addr_t src, dma_addr_t dst, u32 len,
496 u32 ctr1, u32 ctr2, bool is_last, bool is_cyclic)
497 {
498 struct stm32_dma3_hwdesc *hwdesc;
499 dma_addr_t next_lli;
500 u32 next = curr + 1;
501
502 hwdesc = swdesc->lli[curr].hwdesc;
503 hwdesc->ctr1 = ctr1;
504 hwdesc->ctr2 = ctr2;
505 hwdesc->cbr1 = FIELD_PREP(CBR1_BNDT, len);
506 hwdesc->csar = src;
507 hwdesc->cdar = dst;
508
509 if (is_last) {
510 if (is_cyclic)
511 next_lli = swdesc->lli[0].hwdesc_addr;
512 else
513 next_lli = 0;
514 } else {
515 next_lli = swdesc->lli[next].hwdesc_addr;
516 }
517
518 hwdesc->cllr = 0;
519 if (next_lli) {
520 hwdesc->cllr |= CLLR_UT1 | CLLR_UT2 | CLLR_UB1;
521 hwdesc->cllr |= CLLR_USA | CLLR_UDA | CLLR_ULL;
522 hwdesc->cllr |= (next_lli & CLLR_LA);
523 }
524
525 /*
526 * Make sure to flush the CPU's write buffers so that the descriptors are ready to be read
527 * by DMA3. By explicitly using a write memory barrier here, instead of doing it with writel
528 * to enable the channel, we avoid an unnecessary barrier in the case where the descriptors
529 * are reused (DMA_CTRL_REUSE).
530 */
531 if (is_last)
532 dma_wmb();
533 }
534
535 static enum dma_slave_buswidth stm32_dma3_get_max_dw(u32 chan_max_burst,
536 enum stm32_dma3_port_data_width port_max_dw,
537 u32 len, dma_addr_t addr)
538 {
539 enum dma_slave_buswidth max_dw = get_chan_max_dw(port_max_dw, chan_max_burst);
540
541 /* len and addr must be a multiple of dw */
542 return 1 << __ffs(len | addr | max_dw);
543 }
544
545 static u32 stm32_dma3_get_max_burst(u32 len, enum dma_slave_buswidth dw,
546 u32 chan_max_burst, u32 bus_max_burst)
547 {
548 u32 max_burst = chan_max_burst ? chan_max_burst / dw : 1;
549
550 /* len is a multiple of dw, so if len is < chan_max_burst, shorten burst */
551 if (len < chan_max_burst)
552 max_burst = len / dw;
553
554 /*
555 * HW doesn't modify the burst if burst size <= half of the fifo size.
556 * If len is not a multiple of burst size, last burst is shortened by HW.
557 * Take care of maximum burst supported on interconnect bus.
558 */
559 return min_t(u32, max_burst, bus_max_burst);
560 }
561
562 static int stm32_dma3_chan_prep_hw(struct stm32_dma3_chan *chan, enum dma_transfer_direction dir,
563 u32 *ccr, u32 *ctr1, u32 *ctr2,
564 dma_addr_t src_addr, dma_addr_t dst_addr, u32 len)
565 {
566 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
567 struct dma_device dma_device = ddata->dma_dev;
568 u32 src_max_burst = STM32_DMA3_MAX_BURST_LEN, dst_max_burst = STM32_DMA3_MAX_BURST_LEN;
569 u32 sdw, ddw, sbl_max, dbl_max, tcem, init_dw, init_bl_max;
570 u32 _ctr1 = 0, _ctr2 = 0;
571 u32 ch_conf = chan->dt_config.ch_conf;
572 u32 tr_conf = chan->dt_config.tr_conf;
573 u32 sap = FIELD_GET(STM32_DMA3_DT_SAP, tr_conf), sap_max_dw;
574 u32 dap = FIELD_GET(STM32_DMA3_DT_DAP, tr_conf), dap_max_dw;
575
576 dev_dbg(chan2dev(chan), "%s from %pad to %pad\n",
577 dmaengine_get_direction_text(dir), &src_addr, &dst_addr);
578
579 sdw = chan->dma_config.src_addr_width ? : get_chan_max_dw(sap, chan->max_burst);
580 ddw = chan->dma_config.dst_addr_width ? : get_chan_max_dw(dap, chan->max_burst);
581 sbl_max = chan->dma_config.src_maxburst ? : 1;
582 dbl_max = chan->dma_config.dst_maxburst ? : 1;
583
584 /* Following conditions would raise User Setting Error interrupt */
585 if (!(dma_device.src_addr_widths & BIT(sdw)) || !(dma_device.dst_addr_widths & BIT(ddw))) {
586 dev_err(chan2dev(chan), "Bus width (src=%u, dst=%u) not supported\n", sdw, ddw);
587 return -EINVAL;
588 }
589
590 if (ddata->ports_max_dw[1] == DW_INVALID && (sap || dap)) {
591 dev_err(chan2dev(chan), "Only one master port, port 1 is not supported\n");
592 return -EINVAL;
593 }
594
595 sap_max_dw = ddata->ports_max_dw[sap];
596 dap_max_dw = ddata->ports_max_dw[dap];
597 if ((port_is_ahb(sap_max_dw) && sdw == DMA_SLAVE_BUSWIDTH_8_BYTES) ||
598 (port_is_ahb(dap_max_dw) && ddw == DMA_SLAVE_BUSWIDTH_8_BYTES)) {
599 dev_err(chan2dev(chan),
600 "8 bytes buswidth (src=%u, dst=%u) not supported on port (sap=%u, dap=%u\n",
601 sdw, ddw, sap, dap);
602 return -EINVAL;
603 }
604
605 if (FIELD_GET(STM32_DMA3_DT_SINC, tr_conf))
606 _ctr1 |= CTR1_SINC;
607 if (sap)
608 _ctr1 |= CTR1_SAP;
609 if (port_is_axi(sap_max_dw)) /* AXI - apply axi maximum burst limitation */
610 src_max_burst = ddata->axi_max_burst_len;
611 if (FIELD_GET(STM32_DMA3_DT_DINC, tr_conf))
612 _ctr1 |= CTR1_DINC;
613 if (dap)
614 _ctr1 |= CTR1_DAP;
615 if (port_is_axi(dap_max_dw)) /* AXI - apply axi maximum burst limitation */
616 dst_max_burst = ddata->axi_max_burst_len;
617
618 _ctr2 |= FIELD_PREP(CTR2_REQSEL, chan->dt_config.req_line) & ~CTR2_SWREQ;
619 if (FIELD_GET(STM32_DMA3_DT_BREQ, tr_conf))
620 _ctr2 |= CTR2_BREQ;
621 if (dir == DMA_DEV_TO_MEM && FIELD_GET(STM32_DMA3_DT_PFREQ, tr_conf))
622 _ctr2 |= CTR2_PFREQ;
623 tcem = FIELD_GET(STM32_DMA3_DT_TCEM, tr_conf);
624 _ctr2 |= FIELD_PREP(CTR2_TCEM, tcem);
625
626 /* Store TCEM to know on which event TC flag occurred */
627 chan->tcem = tcem;
628 /* Store direction for residue computation */
629 chan->dma_config.direction = dir;
630
631 switch (dir) {
632 case DMA_MEM_TO_DEV:
633 /* Set destination (device) data width and burst */
634 ddw = min_t(u32, ddw, stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw,
635 len, dst_addr));
636 dbl_max = min_t(u32, dbl_max, stm32_dma3_get_max_burst(len, ddw, chan->max_burst,
637 dst_max_burst));
638
639 /* Set source (memory) data width and burst */
640 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);
641 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst);
642 if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf)) {
643 sdw = ddw;
644 sbl_max = dbl_max;
645 }
646
647 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
648 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
649 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
650 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
651
652 if (ddw != sdw) {
653 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
654 /* Should never reach this case as ddw is clamped down */
655 if (len & (ddw - 1)) {
656 dev_err(chan2dev(chan),
657 "Packing mode is enabled and len is not multiple of ddw");
658 return -EINVAL;
659 }
660 }
661
662 /* dst = dev */
663 _ctr2 |= CTR2_DREQ;
664
665 break;
666
667 case DMA_DEV_TO_MEM:
668 /* Set source (device) data width and burst */
669 sdw = min_t(u32, sdw, stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw,
670 len, src_addr));
671 sbl_max = min_t(u32, sbl_max, stm32_dma3_get_max_burst(len, sdw, chan->max_burst,
672 src_max_burst));
673
674 /* Set destination (memory) data width and burst */
675 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);
676 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst);
677 if (!!FIELD_GET(STM32_DMA3_DT_NOPACK, tr_conf) ||
678 ((_ctr2 & CTR2_PFREQ) && ddw > sdw)) { /* Packing to wider ddw not supported */
679 ddw = sdw;
680 dbl_max = sbl_max;
681 }
682
683 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
684 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
685 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
686 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
687
688 if (ddw != sdw) {
689 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
690 /* Should never reach this case as ddw is clamped down */
691 if (len & (ddw - 1)) {
692 dev_err(chan2dev(chan),
693 "Packing mode is enabled and len is not multiple of ddw\n");
694 return -EINVAL;
695 }
696 }
697
698 /* dst = mem */
699 _ctr2 &= ~CTR2_DREQ;
700
701 break;
702
703 case DMA_MEM_TO_MEM:
704 /* Set source (memory) data width and burst */
705 init_dw = sdw;
706 init_bl_max = sbl_max;
707 sdw = stm32_dma3_get_max_dw(chan->max_burst, sap_max_dw, len, src_addr);
708 sbl_max = stm32_dma3_get_max_burst(len, sdw, chan->max_burst, src_max_burst);
709 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {
710 sdw = min_t(u32, init_dw, sdw);
711 sbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, sdw,
712 chan->max_burst,
713 src_max_burst));
714 }
715
716 /* Set destination (memory) data width and burst */
717 init_dw = ddw;
718 init_bl_max = dbl_max;
719 ddw = stm32_dma3_get_max_dw(chan->max_burst, dap_max_dw, len, dst_addr);
720 dbl_max = stm32_dma3_get_max_burst(len, ddw, chan->max_burst, dst_max_burst);
721 if (chan->config_set & STM32_DMA3_CFG_SET_DMA) {
722 ddw = min_t(u32, init_dw, ddw);
723 dbl_max = min_t(u32, init_bl_max, stm32_dma3_get_max_burst(len, ddw,
724 chan->max_burst,
725 dst_max_burst));
726 }
727
728 _ctr1 |= FIELD_PREP(CTR1_SDW_LOG2, ilog2(sdw));
729 _ctr1 |= FIELD_PREP(CTR1_SBL_1, sbl_max - 1);
730 _ctr1 |= FIELD_PREP(CTR1_DDW_LOG2, ilog2(ddw));
731 _ctr1 |= FIELD_PREP(CTR1_DBL_1, dbl_max - 1);
732
733 if (ddw != sdw) {
734 _ctr1 |= FIELD_PREP(CTR1_PAM, CTR1_PAM_PACK_UNPACK);
735 /* Should never reach this case as ddw is clamped down */
736 if (len & (ddw - 1)) {
737 dev_err(chan2dev(chan),
738 "Packing mode is enabled and len is not multiple of ddw");
739 return -EINVAL;
740 }
741 }
742
743 /* CTR2_REQSEL/DREQ/BREQ/PFREQ are ignored with CTR2_SWREQ=1 */
744 _ctr2 |= CTR2_SWREQ;
745
746 break;
747
748 default:
749 dev_err(chan2dev(chan), "Direction %s not supported\n",
750 dmaengine_get_direction_text(dir));
751 return -EINVAL;
752 }
753
754 *ccr |= FIELD_PREP(CCR_PRIO, FIELD_GET(STM32_DMA3_DT_PRIO, ch_conf));
755 *ctr1 = _ctr1;
756 *ctr2 = _ctr2;
757
758 dev_dbg(chan2dev(chan), "%s: sdw=%u bytes sbl=%u beats ddw=%u bytes dbl=%u beats\n",
759 __func__, sdw, sbl_max, ddw, dbl_max);
760
761 return 0;
762 }
763
764 static void stm32_dma3_chan_start(struct stm32_dma3_chan *chan)
765 {
766 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
767 struct virt_dma_desc *vdesc;
768 struct stm32_dma3_hwdesc *hwdesc;
769 u32 id = chan->id;
770 u32 csr, ccr;
771
772 vdesc = vchan_next_desc(&chan->vchan);
773 if (!vdesc) {
774 chan->swdesc = NULL;
775 return;
776 }
777 list_del(&vdesc->node);
778
779 chan->swdesc = to_stm32_dma3_swdesc(vdesc);
780 hwdesc = chan->swdesc->lli[0].hwdesc;
781
782 stm32_dma3_chan_dump_hwdesc(chan, chan->swdesc);
783
784 writel_relaxed(chan->swdesc->ccr, ddata->base + STM32_DMA3_CCR(id));
785 writel_relaxed(hwdesc->ctr1, ddata->base + STM32_DMA3_CTR1(id));
786 writel_relaxed(hwdesc->ctr2, ddata->base + STM32_DMA3_CTR2(id));
787 writel_relaxed(hwdesc->cbr1, ddata->base + STM32_DMA3_CBR1(id));
788 writel_relaxed(hwdesc->csar, ddata->base + STM32_DMA3_CSAR(id));
789 writel_relaxed(hwdesc->cdar, ddata->base + STM32_DMA3_CDAR(id));
790 writel_relaxed(hwdesc->cllr, ddata->base + STM32_DMA3_CLLR(id));
791
792 /* Clear any pending interrupts */
793 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(id));
794 if (csr & CSR_ALL_F)
795 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(id));
796
797 stm32_dma3_chan_dump_reg(chan);
798
799 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(id));
800 writel_relaxed(ccr | CCR_EN, ddata->base + STM32_DMA3_CCR(id));
801
802 chan->dma_status = DMA_IN_PROGRESS;
803
804 dev_dbg(chan2dev(chan), "vchan %pK: started\n", &chan->vchan);
805 }
806
807 static int stm32_dma3_chan_suspend(struct stm32_dma3_chan *chan, bool susp)
808 {
809 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
810 u32 csr, ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;
811 int ret = 0;
812
813 if (susp)
814 ccr |= CCR_SUSP;
815 else
816 ccr &= ~CCR_SUSP;
817
818 writel_relaxed(ccr, ddata->base + STM32_DMA3_CCR(chan->id));
819
820 if (susp) {
821 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,
822 csr & CSR_SUSPF, 1, 10);
823 if (!ret)
824 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
825
826 stm32_dma3_chan_dump_reg(chan);
827 }
828
829 return ret;
830 }
831
832 static void stm32_dma3_chan_reset(struct stm32_dma3_chan *chan)
833 {
834 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
835 u32 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & ~CCR_EN;
836
837 writel_relaxed(ccr |= CCR_RESET, ddata->base + STM32_DMA3_CCR(chan->id));
838 }
839
840 static int stm32_dma3_chan_get_curr_hwdesc(struct stm32_dma3_swdesc *swdesc, u32 cllr, u32 *residue)
841 {
842 u32 i, lli_offset, next_lli_offset = cllr & CLLR_LA;
843
844 /* If cllr is null, it means it is either the last or single item */
845 if (!cllr)
846 return swdesc->lli_size - 1;
847
848 /* In cyclic mode, go fast and first check we are not on the last item */
849 if (swdesc->cyclic && next_lli_offset == (swdesc->lli[0].hwdesc_addr & CLLR_LA))
850 return swdesc->lli_size - 1;
851
852 /* As transfer is in progress, look backward from the last item */
853 for (i = swdesc->lli_size - 1; i > 0; i--) {
854 *residue += FIELD_GET(CBR1_BNDT, swdesc->lli[i].hwdesc->cbr1);
855 lli_offset = swdesc->lli[i].hwdesc_addr & CLLR_LA;
856 if (lli_offset == next_lli_offset)
857 return i - 1;
858 }
859
860 return -EINVAL;
861 }
862
863 static void stm32_dma3_chan_set_residue(struct stm32_dma3_chan *chan,
864 struct stm32_dma3_swdesc *swdesc,
865 struct dma_tx_state *txstate)
866 {
867 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
868 struct device *dev = chan2dev(chan);
869 struct stm32_dma3_hwdesc *hwdesc;
870 u32 residue, curr_lli, csr, cdar, cbr1, cllr, bndt, fifol;
871 bool pack_unpack;
872 int ret;
873
874 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));
875 if (!(csr & CSR_IDLEF) && chan->dma_status != DMA_PAUSED) {
876 /* Suspend current transfer to read registers for a snapshot */
877 writel_relaxed(swdesc->ccr | CCR_SUSP, ddata->base + STM32_DMA3_CCR(chan->id));
878 ret = readl_relaxed_poll_timeout_atomic(ddata->base + STM32_DMA3_CSR(chan->id), csr,
879 csr & (CSR_SUSPF | CSR_IDLEF), 1, 10);
880
881 if (ret || ((csr & CSR_TCF) && (csr & CSR_IDLEF))) {
882 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
883 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));
884 if (ret)
885 dev_err(dev, "Channel suspension timeout, csr=%08x\n", csr);
886 }
887 }
888
889 /* If channel is still active (CSR_IDLEF is not set), can't get a reliable residue */
890 if (!(csr & CSR_IDLEF))
891 dev_warn(dev, "Can't get residue: channel still active, csr=%08x\n", csr);
892
893 /*
894 * If channel is not suspended, but Idle and Transfer Complete are set,
895 * linked-list is over, no residue
896 */
897 if (!(csr & CSR_SUSPF) && (csr & CSR_TCF) && (csr & CSR_IDLEF))
898 return;
899
900 /* Read registers to have a snapshot */
901 cllr = readl_relaxed(ddata->base + STM32_DMA3_CLLR(chan->id));
902 cbr1 = readl_relaxed(ddata->base + STM32_DMA3_CBR1(chan->id));
903 cdar = readl_relaxed(ddata->base + STM32_DMA3_CDAR(chan->id));
904
905 /* Resume current transfer */
906 if (csr & CSR_SUSPF) {
907 writel_relaxed(CFCR_SUSPF, ddata->base + STM32_DMA3_CFCR(chan->id));
908 writel_relaxed(swdesc->ccr, ddata->base + STM32_DMA3_CCR(chan->id));
909 }
910
911 /* Add current BNDT */
912 bndt = FIELD_GET(CBR1_BNDT, cbr1);
913 residue = bndt;
914
915 /* Get current hwdesc and cumulate residue of pending hwdesc BNDT */
916 ret = stm32_dma3_chan_get_curr_hwdesc(swdesc, cllr, &residue);
917 if (ret < 0) {
918 dev_err(chan2dev(chan), "Can't get residue: current hwdesc not found\n");
919 return;
920 }
921 curr_lli = ret;
922
923 /* Read current FIFO level - in units of programmed destination data width */
924 hwdesc = swdesc->lli[curr_lli].hwdesc;
925 fifol = FIELD_GET(CSR_FIFOL, csr) * (1 << FIELD_GET(CTR1_DDW_LOG2, hwdesc->ctr1));
926 /* If the FIFO contains as many bytes as its size, it can't contain more */
927 if (fifol == (1 << (chan->fifo_size + 1)))
928 goto skip_fifol_update;
929
930 /*
931 * In case of PACKING (Destination burst length > Source burst length) or UNPACKING
932 * (Source burst length > Destination burst length), bytes could be pending in the FIFO
933 * (to be packed up to Destination burst length or unpacked into Destination burst length
934 * chunks).
935 * BNDT is not reliable, as it reflects the number of bytes read from the source but not the
936 * number of bytes written to the destination.
937 * FIFOL is also not sufficient, because it reflects the number of available write beats in
938 * units of Destination data width but not the bytes not yet packed or unpacked.
939 * In case of Destination increment DINC, it is possible to compute the number of bytes in
940 * the FIFO:
941 * fifol_in_bytes = bytes_read - bytes_written.
942 */
943 pack_unpack = !!(FIELD_GET(CTR1_PAM, hwdesc->ctr1) == CTR1_PAM_PACK_UNPACK);
944 if (pack_unpack && (hwdesc->ctr1 & CTR1_DINC)) {
945 int bytes_read = FIELD_GET(CBR1_BNDT, hwdesc->cbr1) - bndt;
946 int bytes_written = cdar - hwdesc->cdar;
947
948 if (bytes_read > 0)
949 fifol = bytes_read - bytes_written;
950 }
951
952 skip_fifol_update:
953 if (fifol) {
954 dev_dbg(chan2dev(chan), "%u byte(s) in the FIFO\n", fifol);
955 dma_set_in_flight_bytes(txstate, fifol);
956 /*
957 * Residue is already accurate for DMA_MEM_TO_DEV as BNDT reflects data read from
958 * the source memory buffer, so just need to add fifol to residue in case of
959 * DMA_DEV_TO_MEM transfer because these bytes are not yet written in destination
960 * memory buffer.
961 */
962 if (chan->dma_config.direction == DMA_DEV_TO_MEM)
963 residue += fifol;
964 }
965 dma_set_residue(txstate, residue);
966 }
967
968 static int stm32_dma3_chan_stop(struct stm32_dma3_chan *chan)
969 {
970 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
971 u32 ccr;
972 int ret = 0;
973
974 chan->dma_status = DMA_COMPLETE;
975
976 /* Disable interrupts */
977 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id));
978 writel_relaxed(ccr & ~(CCR_ALLIE | CCR_EN), ddata->base + STM32_DMA3_CCR(chan->id));
979
980 if (!(ccr & CCR_SUSP) && (ccr & CCR_EN)) {
981 /* Suspend the channel */
982 ret = stm32_dma3_chan_suspend(chan, true);
983 if (ret)
984 dev_warn(chan2dev(chan), "%s: timeout, data might be lost\n", __func__);
985 }
986
987 /*
988 * Reset the channel: this causes the reset of the FIFO and the reset of the channel
989 * internal state, the reset of CCR_EN and CCR_SUSP bits.
990 */
991 stm32_dma3_chan_reset(chan);
992
993 return ret;
994 }
995
996 static void stm32_dma3_chan_complete(struct stm32_dma3_chan *chan)
997 {
998 if (!chan->swdesc)
999 return;
1000
1001 vchan_cookie_complete(&chan->swdesc->vdesc);
1002 chan->swdesc = NULL;
1003 stm32_dma3_chan_start(chan);
1004 }
1005
1006 static irqreturn_t stm32_dma3_chan_irq(int irq, void *devid)
1007 {
1008 struct stm32_dma3_chan *chan = devid;
1009 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1010 u32 misr, csr, ccr;
1011
1012 spin_lock(&chan->vchan.lock);
1013
1014 misr = readl_relaxed(ddata->base + STM32_DMA3_MISR);
1015 if (!(misr & MISR_MIS(chan->id))) {
1016 spin_unlock(&chan->vchan.lock);
1017 return IRQ_NONE;
1018 }
1019
1020 csr = readl_relaxed(ddata->base + STM32_DMA3_CSR(chan->id));
1021 ccr = readl_relaxed(ddata->base + STM32_DMA3_CCR(chan->id)) & CCR_ALLIE;
1022
1023 if (csr & CSR_TCF && ccr & CCR_TCIE) {
1024 if (chan->swdesc->cyclic)
1025 vchan_cyclic_callback(&chan->swdesc->vdesc);
1026 else
1027 stm32_dma3_chan_complete(chan);
1028 }
1029
1030 if (csr & CSR_USEF && ccr & CCR_USEIE) {
1031 dev_err(chan2dev(chan), "User setting error\n");
1032 chan->dma_status = DMA_ERROR;
1033 /* CCR.EN automatically cleared by HW */
1034 stm32_dma3_check_user_setting(chan);
1035 stm32_dma3_chan_reset(chan);
1036 }
1037
1038 if (csr & CSR_ULEF && ccr & CCR_ULEIE) {
1039 dev_err(chan2dev(chan), "Update link transfer error\n");
1040 chan->dma_status = DMA_ERROR;
1041 /* CCR.EN automatically cleared by HW */
1042 stm32_dma3_chan_reset(chan);
1043 }
1044
1045 if (csr & CSR_DTEF && ccr & CCR_DTEIE) {
1046 dev_err(chan2dev(chan), "Data transfer error\n");
1047 chan->dma_status = DMA_ERROR;
1048 /* CCR.EN automatically cleared by HW */
1049 stm32_dma3_chan_reset(chan);
1050 }
1051
1052 /*
1053 * Half Transfer Interrupt may be disabled but Half Transfer Flag can be set,
1054 * ensure HTF flag to be cleared, with other flags.
1055 */
1056 csr &= (ccr | CCR_HTIE);
1057
1058 if (csr)
1059 writel_relaxed(csr, ddata->base + STM32_DMA3_CFCR(chan->id));
1060
1061 spin_unlock(&chan->vchan.lock);
1062
1063 return IRQ_HANDLED;
1064 }
1065
1066 static int stm32_dma3_alloc_chan_resources(struct dma_chan *c)
1067 {
1068 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1069 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1070 u32 id = chan->id, csemcr, ccid;
1071 int ret;
1072
1073 ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
1074 if (ret < 0)
1075 return ret;
1076
1077 /* Ensure the channel is free */
1078 if (chan->semaphore_mode &&
1079 readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id)) & CSEMCR_SEM_MUTEX) {
1080 ret = -EBUSY;
1081 goto err_put_sync;
1082 }
1083
1084 chan->lli_pool = dmam_pool_create(dev_name(&c->dev->device), c->device->dev,
1085 sizeof(struct stm32_dma3_hwdesc),
1086 __alignof__(struct stm32_dma3_hwdesc), SZ_64K);
1087 if (!chan->lli_pool) {
1088 dev_err(chan2dev(chan), "Failed to create LLI pool\n");
1089 ret = -ENOMEM;
1090 goto err_put_sync;
1091 }
1092
1093 /* Take the channel semaphore */
1094 if (chan->semaphore_mode) {
1095 writel_relaxed(CSEMCR_SEM_MUTEX, ddata->base + STM32_DMA3_CSEMCR(id));
1096 csemcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(id));
1097 ccid = FIELD_GET(CSEMCR_SEM_CCID, csemcr);
1098 /* Check that the channel is well taken */
1099 if (ccid != CCIDCFGR_CID1) {
1100 dev_err(chan2dev(chan), "Not under CID1 control (in-use by CID%d)\n", ccid);
1101 ret = -EPERM;
1102 goto err_pool_destroy;
1103 }
1104 dev_dbg(chan2dev(chan), "Under CID1 control (semcr=0x%08x)\n", csemcr);
1105 }
1106
1107 return 0;
1108
1109 err_pool_destroy:
1110 dmam_pool_destroy(chan->lli_pool);
1111 chan->lli_pool = NULL;
1112
1113 err_put_sync:
1114 pm_runtime_put_sync(ddata->dma_dev.dev);
1115
1116 return ret;
1117 }
1118
1119 static void stm32_dma3_free_chan_resources(struct dma_chan *c)
1120 {
1121 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1122 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1123 unsigned long flags;
1124
1125 /* Ensure channel is in idle state */
1126 spin_lock_irqsave(&chan->vchan.lock, flags);
1127 stm32_dma3_chan_stop(chan);
1128 chan->swdesc = NULL;
1129 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1130
1131 vchan_free_chan_resources(to_virt_chan(c));
1132
1133 dmam_pool_destroy(chan->lli_pool);
1134 chan->lli_pool = NULL;
1135
1136 /* Release the channel semaphore */
1137 if (chan->semaphore_mode)
1138 writel_relaxed(0, ddata->base + STM32_DMA3_CSEMCR(chan->id));
1139
1140 pm_runtime_put_sync(ddata->dma_dev.dev);
1141
1142 /* Reset configuration */
1143 memset(&chan->dt_config, 0, sizeof(chan->dt_config));
1144 memset(&chan->dma_config, 0, sizeof(chan->dma_config));
1145 chan->config_set = 0;
1146 }
1147
1148 static u32 stm32_dma3_get_ll_count(struct stm32_dma3_chan *chan, size_t len, bool prevent_refactor)
1149 {
1150 u32 count;
1151
1152 if (prevent_refactor)
1153 return DIV_ROUND_UP(len, STM32_DMA3_MAX_BLOCK_SIZE);
1154
1155 count = len / STM32_DMA3_MAX_BLOCK_SIZE;
1156 len -= (len / STM32_DMA3_MAX_BLOCK_SIZE) * STM32_DMA3_MAX_BLOCK_SIZE;
1157
1158 if (len >= chan->max_burst) {
1159 count += 1; /* len < STM32_DMA3_MAX_BLOCK_SIZE here, so it fits in one item */
1160 len -= (len / chan->max_burst) * chan->max_burst;
1161 }
1162
1163 /* Unaligned remainder fits in one extra item */
1164 if (len > 0)
1165 count += 1;
1166
1167 return count;
1168 }
1169
1170 static void stm32_dma3_init_chan_config_for_memcpy(struct stm32_dma3_chan *chan,
1171 dma_addr_t dst, dma_addr_t src)
1172 {
1173 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1174 u32 dw = get_chan_max_dw(ddata->ports_max_dw[0], chan->max_burst); /* port 0 by default */
1175 u32 burst = chan->max_burst / dw;
1176
1177 /* Initialize dt_config if channel not pre-configured through DT */
1178 if (!(chan->config_set & STM32_DMA3_CFG_SET_DT)) {
1179 chan->dt_config.ch_conf = FIELD_PREP(STM32_DMA3_DT_PRIO, CCR_PRIO_VERY_HIGH);
1180 chan->dt_config.ch_conf |= FIELD_PREP(STM32_DMA3_DT_FIFO, chan->fifo_size);
1181 chan->dt_config.tr_conf = STM32_DMA3_DT_SINC | STM32_DMA3_DT_DINC;
1182 chan->dt_config.tr_conf |= FIELD_PREP(STM32_DMA3_DT_TCEM, CTR2_TCEM_CHANNEL);
1183 }
1184
1185 /* Initialize dma_config if dmaengine_slave_config() not used */
1186 if (!(chan->config_set & STM32_DMA3_CFG_SET_DMA)) {
1187 chan->dma_config.src_addr_width = dw;
1188 chan->dma_config.dst_addr_width = dw;
1189 chan->dma_config.src_maxburst = burst;
1190 chan->dma_config.dst_maxburst = burst;
1191 chan->dma_config.src_addr = src;
1192 chan->dma_config.dst_addr = dst;
1193 }
1194 }
1195
1196 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_memcpy(struct dma_chan *c,
1197 dma_addr_t dst, dma_addr_t src,
1198 size_t len, unsigned long flags)
1199 {
1200 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1201 struct stm32_dma3_swdesc *swdesc;
1202 size_t next_size, offset;
1203 u32 count, i, ctr1, ctr2;
1204 bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) ||
1205 !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf);
1206
1207 count = stm32_dma3_get_ll_count(chan, len, prevent_refactor);
1208
1209 swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1210 if (!swdesc)
1211 return NULL;
1212
1213 if (chan->config_set != STM32_DMA3_CFG_SET_BOTH)
1214 stm32_dma3_init_chan_config_for_memcpy(chan, dst, src);
1215
1216 for (i = 0, offset = 0; offset < len; i++, offset += next_size) {
1217 size_t remaining;
1218 int ret;
1219
1220 remaining = len - offset;
1221 next_size = min_t(size_t, remaining, STM32_DMA3_MAX_BLOCK_SIZE);
1222
1223 if (!prevent_refactor &&
1224 (next_size < STM32_DMA3_MAX_BLOCK_SIZE && next_size >= chan->max_burst))
1225 next_size = chan->max_burst * (remaining / chan->max_burst);
1226
1227 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
1228 src + offset, dst + offset, next_size);
1229 if (ret)
1230 goto err_desc_free;
1231
1232 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src + offset, dst + offset, next_size,
1233 ctr1, ctr2, next_size == remaining, false);
1234 }
1235
1236 /* Enable Errors interrupts */
1237 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1238 /* Enable Transfer state interrupts */
1239 swdesc->ccr |= CCR_TCIE;
1240
1241 swdesc->cyclic = false;
1242
1243 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1244
1245 err_desc_free:
1246 stm32_dma3_chan_desc_free(chan, swdesc);
1247
1248 return NULL;
1249 }
1250
1251 static struct dma_async_tx_descriptor *stm32_dma3_prep_slave_sg(struct dma_chan *c,
1252 struct scatterlist *sgl,
1253 unsigned int sg_len,
1254 enum dma_transfer_direction dir,
1255 unsigned long flags, void *context)
1256 {
1257 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1258 struct stm32_dma3_swdesc *swdesc;
1259 struct scatterlist *sg;
1260 size_t len;
1261 dma_addr_t sg_addr, dev_addr, src, dst;
1262 u32 i, j, count, ctr1, ctr2;
1263 bool prevent_refactor = !!FIELD_GET(STM32_DMA3_DT_NOPACK, chan->dt_config.tr_conf) ||
1264 !!FIELD_GET(STM32_DMA3_DT_NOREFACT, chan->dt_config.tr_conf);
1265 int ret;
1266
1267 count = 0;
1268 for_each_sg(sgl, sg, sg_len, i)
1269 count += stm32_dma3_get_ll_count(chan, sg_dma_len(sg), prevent_refactor);
1270
1271 swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1272 if (!swdesc)
1273 return NULL;
1274
1275 /* sg_len and i correspond to the initial sgl; count and j correspond to the hwdesc LL */
1276 j = 0;
1277 for_each_sg(sgl, sg, sg_len, i) {
1278 sg_addr = sg_dma_address(sg);
1279 dev_addr = (dir == DMA_MEM_TO_DEV) ? chan->dma_config.dst_addr :
1280 chan->dma_config.src_addr;
1281 len = sg_dma_len(sg);
1282
1283 do {
1284 size_t chunk = min_t(size_t, len, STM32_DMA3_MAX_BLOCK_SIZE);
1285
1286 if (!prevent_refactor &&
1287 (chunk < STM32_DMA3_MAX_BLOCK_SIZE && chunk >= chan->max_burst))
1288 chunk = chan->max_burst * (len / chan->max_burst);
1289
1290 if (dir == DMA_MEM_TO_DEV) {
1291 src = sg_addr;
1292 dst = dev_addr;
1293
1294 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,
1295 src, dst, chunk);
1296
1297 if (FIELD_GET(CTR1_DINC, ctr1))
1298 dev_addr += chunk;
1299 } else { /* (dir == DMA_DEV_TO_MEM || dir == DMA_MEM_TO_MEM) */
1300 src = dev_addr;
1301 dst = sg_addr;
1302
1303 ret = stm32_dma3_chan_prep_hw(chan, dir, &swdesc->ccr, &ctr1, &ctr2,
1304 src, dst, chunk);
1305
1306 if (FIELD_GET(CTR1_SINC, ctr1))
1307 dev_addr += chunk;
1308 }
1309
1310 if (ret)
1311 goto err_desc_free;
1312
1313 stm32_dma3_chan_prep_hwdesc(chan, swdesc, j, src, dst, chunk,
1314 ctr1, ctr2, j == (count - 1), false);
1315
1316 sg_addr += chunk;
1317 len -= chunk;
1318 j++;
1319 } while (len);
1320 }
1321
1322 if (count != sg_len && chan->tcem != CTR2_TCEM_CHANNEL)
1323 dev_warn(chan2dev(chan), "Linked-list refactored, %d items instead of %d\n",
1324 count, sg_len);
1325
1326 /* Enable Error interrupts */
1327 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1328 /* Enable Transfer state interrupts */
1329 swdesc->ccr |= CCR_TCIE;
1330
1331 swdesc->cyclic = false;
1332
1333 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1334
1335 err_desc_free:
1336 stm32_dma3_chan_desc_free(chan, swdesc);
1337
1338 return NULL;
1339 }
1340
1341 static struct dma_async_tx_descriptor *stm32_dma3_prep_dma_cyclic(struct dma_chan *c,
1342 dma_addr_t buf_addr,
1343 size_t buf_len, size_t period_len,
1344 enum dma_transfer_direction dir,
1345 unsigned long flags)
1346 {
1347 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1348 struct stm32_dma3_swdesc *swdesc;
1349 dma_addr_t src, dst;
1350 u32 count, i, ctr1, ctr2;
1351 int ret;
1352
1353 if (!buf_len || !period_len || period_len > STM32_DMA3_MAX_BLOCK_SIZE) {
1354 dev_err(chan2dev(chan), "Invalid buffer/period length\n");
1355 return NULL;
1356 }
1357
1358 if (buf_len % period_len) {
1359 dev_err(chan2dev(chan), "Buffer length not multiple of period length\n");
1360 return NULL;
1361 }
1362
1363 count = buf_len / period_len;
1364 swdesc = stm32_dma3_chan_desc_alloc(chan, count);
1365 if (!swdesc)
1366 return NULL;
1367
1368 if (dir == DMA_MEM_TO_DEV) {
1369 src = buf_addr;
1370 dst = chan->dma_config.dst_addr;
1371
1372 ret = stm32_dma3_chan_prep_hw(chan, DMA_MEM_TO_DEV, &swdesc->ccr, &ctr1, &ctr2,
1373 src, dst, period_len);
1374 } else if (dir == DMA_DEV_TO_MEM) {
1375 src = chan->dma_config.src_addr;
1376 dst = buf_addr;
1377
1378 ret = stm32_dma3_chan_prep_hw(chan, DMA_DEV_TO_MEM, &swdesc->ccr, &ctr1, &ctr2,
1379 src, dst, period_len);
1380 } else {
1381 dev_err(chan2dev(chan), "Invalid direction\n");
1382 ret = -EINVAL;
1383 }
1384
1385 if (ret)
1386 goto err_desc_free;
1387
1388 for (i = 0; i < count; i++) {
1389 if (dir == DMA_MEM_TO_DEV) {
1390 src = buf_addr + i * period_len;
1391 dst = chan->dma_config.dst_addr;
1392 } else { /* (dir == DMA_DEV_TO_MEM) */
1393 src = chan->dma_config.src_addr;
1394 dst = buf_addr + i * period_len;
1395 }
1396
1397 stm32_dma3_chan_prep_hwdesc(chan, swdesc, i, src, dst, period_len,
1398 ctr1, ctr2, i == (count - 1), true);
1399 }
1400
1401 /* Enable Error interrupts */
1402 swdesc->ccr |= CCR_USEIE | CCR_ULEIE | CCR_DTEIE;
1403 /* Enable Transfer state interrupts */
1404 swdesc->ccr |= CCR_TCIE;
1405
1406 swdesc->cyclic = true;
1407
1408 return vchan_tx_prep(&chan->vchan, &swdesc->vdesc, flags);
1409
1410 err_desc_free:
1411 stm32_dma3_chan_desc_free(chan, swdesc);
1412
1413 return NULL;
1414 }
1415
1416 static void stm32_dma3_caps(struct dma_chan *c, struct dma_slave_caps *caps)
1417 {
1418 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1419
1420 if (!chan->fifo_size) {
1421 caps->max_burst = 0;
1422 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1423 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1424 } else {
1425 /* Burst transfer should not exceed half of the fifo size */
1426 caps->max_burst = chan->max_burst;
1427 if (caps->max_burst < DMA_SLAVE_BUSWIDTH_8_BYTES) {
1428 caps->src_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1429 caps->dst_addr_widths &= ~BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1430 }
1431 }
1432 }
1433
1434 static int stm32_dma3_config(struct dma_chan *c, struct dma_slave_config *config)
1435 {
1436 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1437
1438 memcpy(&chan->dma_config, config, sizeof(*config));
1439 chan->config_set |= STM32_DMA3_CFG_SET_DMA;
1440
1441 return 0;
1442 }
1443
1444 static int stm32_dma3_pause(struct dma_chan *c)
1445 {
1446 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1447 int ret;
1448
1449 ret = stm32_dma3_chan_suspend(chan, true);
1450 if (ret)
1451 return ret;
1452
1453 chan->dma_status = DMA_PAUSED;
1454
1455 dev_dbg(chan2dev(chan), "vchan %pK: paused\n", &chan->vchan);
1456
1457 return 0;
1458 }
1459
1460 static int stm32_dma3_resume(struct dma_chan *c)
1461 {
1462 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1463
1464 stm32_dma3_chan_suspend(chan, false);
1465
1466 chan->dma_status = DMA_IN_PROGRESS;
1467
1468 dev_dbg(chan2dev(chan), "vchan %pK: resumed\n", &chan->vchan);
1469
1470 return 0;
1471 }
1472
1473 static int stm32_dma3_terminate_all(struct dma_chan *c)
1474 {
1475 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1476 unsigned long flags;
1477 LIST_HEAD(head);
1478
1479 spin_lock_irqsave(&chan->vchan.lock, flags);
1480
1481 if (chan->swdesc) {
1482 vchan_terminate_vdesc(&chan->swdesc->vdesc);
1483 chan->swdesc = NULL;
1484 }
1485
1486 stm32_dma3_chan_stop(chan);
1487
1488 vchan_get_all_descriptors(&chan->vchan, &head);
1489
1490 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1491 vchan_dma_desc_free_list(&chan->vchan, &head);
1492
1493 dev_dbg(chan2dev(chan), "vchan %pK: terminated\n", &chan->vchan);
1494
1495 return 0;
1496 }
1497
1498 static void stm32_dma3_synchronize(struct dma_chan *c)
1499 {
1500 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1501
1502 vchan_synchronize(&chan->vchan);
1503 }
1504
1505 static enum dma_status stm32_dma3_tx_status(struct dma_chan *c, dma_cookie_t cookie,
1506 struct dma_tx_state *txstate)
1507 {
1508 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1509 struct stm32_dma3_swdesc *swdesc = NULL;
1510 enum dma_status status;
1511 unsigned long flags;
1512 struct virt_dma_desc *vd;
1513
1514 status = dma_cookie_status(c, cookie, txstate);
1515 if (status == DMA_COMPLETE)
1516 return status;
1517
1518 if (!txstate)
1519 return chan->dma_status;
1520
1521 spin_lock_irqsave(&chan->vchan.lock, flags);
1522
1523 vd = vchan_find_desc(&chan->vchan, cookie);
1524 if (vd)
1525 swdesc = to_stm32_dma3_swdesc(vd);
1526 else if (chan->swdesc && chan->swdesc->vdesc.tx.cookie == cookie)
1527 swdesc = chan->swdesc;
1528
1529 /* Get residue/in_flight_bytes only if a transfer is currently running (swdesc != NULL) */
1530 if (swdesc)
1531 stm32_dma3_chan_set_residue(chan, swdesc, txstate);
1532
1533 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1534
1535 return chan->dma_status;
1536 }
1537
1538 static void stm32_dma3_issue_pending(struct dma_chan *c)
1539 {
1540 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1541 unsigned long flags;
1542
1543 spin_lock_irqsave(&chan->vchan.lock, flags);
1544
1545 if (vchan_issue_pending(&chan->vchan) && !chan->swdesc) {
1546 dev_dbg(chan2dev(chan), "vchan %pK: issued\n", &chan->vchan);
1547 stm32_dma3_chan_start(chan);
1548 }
1549
1550 spin_unlock_irqrestore(&chan->vchan.lock, flags);
1551 }
1552
1553 static bool stm32_dma3_filter_fn(struct dma_chan *c, void *fn_param)
1554 {
1555 struct stm32_dma3_chan *chan = to_stm32_dma3_chan(c);
1556 struct stm32_dma3_ddata *ddata = to_stm32_dma3_ddata(chan);
1557 struct stm32_dma3_dt_conf *conf = fn_param;
1558 u32 mask, semcr;
1559 int ret;
1560
1561 dev_dbg(c->device->dev, "%s(%s): req_line=%d ch_conf=%08x tr_conf=%08x\n",
1562 __func__, dma_chan_name(c), conf->req_line, conf->ch_conf, conf->tr_conf);
1563
1564 if (!of_property_read_u32(c->device->dev->of_node, "dma-channel-mask", &mask))
1565 if (!(mask & BIT(chan->id)))
1566 return false;
1567
1568 ret = pm_runtime_resume_and_get(ddata->dma_dev.dev);
1569 if (ret < 0)
1570 return false;
1571 semcr = readl_relaxed(ddata->base + STM32_DMA3_CSEMCR(chan->id));
1572 pm_runtime_put_sync(ddata->dma_dev.dev);
1573
1574 /* Check if chan is free */
1575 if (semcr & CSEMCR_SEM_MUTEX)
1576 return false;
1577
1578 /* Check if chan fifo fits well */
1579 if (FIELD_GET(STM32_DMA3_DT_FIFO, conf->ch_conf) != chan->fifo_size)
1580 return false;
1581
1582 return true;
1583 }
1584
1585 static struct dma_chan *stm32_dma3_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma)
1586 {
1587 struct stm32_dma3_ddata *ddata = ofdma->of_dma_data;
1588 dma_cap_mask_t mask = ddata->dma_dev.cap_mask;
1589 struct stm32_dma3_dt_conf conf;
1590 struct stm32_dma3_chan *chan;
1591 struct dma_chan *c;
1592
1593 if (dma_spec->args_count < 3) {
1594 dev_err(ddata->dma_dev.dev, "Invalid args count\n");
1595 return NULL;
1596 }
1597
1598 conf.req_line = dma_spec->args[0];
1599 conf.ch_conf = dma_spec->args[1];
1600 conf.tr_conf = dma_spec->args[2];
1601
1602 if (conf.req_line >= ddata->dma_requests) {
1603 dev_err(ddata->dma_dev.dev, "Invalid request line\n");
1604 return NULL;
1605 }
1606
1607 /* Request dma channel among the generic dma controller list */
1608 c = dma_request_channel(mask, stm32_dma3_filter_fn, &conf);
1609 if (!c) {
1610 dev_err(ddata->dma_dev.dev, "No suitable channel found\n");
1611 return NULL;
1612 }
1613
1614 chan = to_stm32_dma3_chan(c);
1615 chan->dt_config = conf;
1616 chan->config_set |= STM32_DMA3_CFG_SET_DT;
1617
1618 return c;
1619 }
1620
1621 static u32 stm32_dma3_check_rif(struct stm32_dma3_ddata *ddata)
1622 {
1623 u32 chan_reserved, mask = 0, i, ccidcfgr, invalid_cid = 0;
1624
1625 /* Reserve Secure channels */
1626 chan_reserved = readl_relaxed(ddata->base + STM32_DMA3_SECCFGR);
1627
1628 /*
1629 * CID filtering must be configured to ensure that the DMA3 channel will inherit the CID of
1630 * the processor which is configuring and using the given channel.
1631 * In case CID filtering is not configured, dma-channel-mask property can be used to
1632 * specify available DMA channels to the kernel.
1633 */
1634 of_property_read_u32(ddata->dma_dev.dev->of_node, "dma-channel-mask", &mask);
1635
1636 /* Reserve !CID-filtered not in dma-channel-mask, static CID != CID1, CID1 not allowed */
1637 for (i = 0; i < ddata->dma_channels; i++) {
1638 ccidcfgr = readl_relaxed(ddata->base + STM32_DMA3_CCIDCFGR(i));
1639
1640 if (!(ccidcfgr & CCIDCFGR_CFEN)) { /* !CID-filtered */
1641 invalid_cid |= BIT(i);
1642 if (!(mask & BIT(i))) /* Not in dma-channel-mask */
1643 chan_reserved |= BIT(i);
1644 } else { /* CID-filtered */
1645 if (!(ccidcfgr & CCIDCFGR_SEM_EN)) { /* Static CID mode */
1646 if (FIELD_GET(CCIDCFGR_SCID, ccidcfgr) != CCIDCFGR_CID1)
1647 chan_reserved |= BIT(i);
1648 } else { /* Semaphore mode */
1649 if (!FIELD_GET(CCIDCFGR_SEM_WLIST_CID1, ccidcfgr))
1650 chan_reserved |= BIT(i);
1651 ddata->chans[i].semaphore_mode = true;
1652 }
1653 }
1654 dev_dbg(ddata->dma_dev.dev, "chan%d: %s mode, %s\n", i,
1655 !(ccidcfgr & CCIDCFGR_CFEN) ? "!CID-filtered" :
1656 ddata->chans[i].semaphore_mode ? "Semaphore" : "Static CID",
1657 (chan_reserved & BIT(i)) ? "denied" :
1658 mask & BIT(i) ? "force allowed" : "allowed");
1659 }
1660
1661 if (invalid_cid)
1662 dev_warn(ddata->dma_dev.dev, "chan%*pbl have invalid CID configuration\n",
1663 ddata->dma_channels, &invalid_cid);
1664
1665 return chan_reserved;
1666 }
1667
1668 static struct stm32_dma3_pdata stm32mp25_pdata = {
1669 .axi_max_burst_len = 16,
1670 };
1671
1672 static const struct of_device_id stm32_dma3_of_match[] = {
1673 { .compatible = "st,stm32mp25-dma3", .data = &stm32mp25_pdata, },
1674 { /* sentinel */ },
1675 };
1676 MODULE_DEVICE_TABLE(of, stm32_dma3_of_match);
1677
1678 static int stm32_dma3_probe(struct platform_device *pdev)
1679 {
1680 struct device_node *np = pdev->dev.of_node;
1681 const struct stm32_dma3_pdata *pdata;
1682 struct stm32_dma3_ddata *ddata;
1683 struct reset_control *reset;
1684 struct stm32_dma3_chan *chan;
1685 struct dma_device *dma_dev;
1686 u32 master_ports, chan_reserved, i, verr;
1687 u64 hwcfgr;
1688 int ret;
1689
1690 ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
1691 if (!ddata)
1692 return -ENOMEM;
1693 platform_set_drvdata(pdev, ddata);
1694
1695 dma_dev = &ddata->dma_dev;
1696
1697 ddata->base = devm_platform_ioremap_resource(pdev, 0);
1698 if (IS_ERR(ddata->base))
1699 return PTR_ERR(ddata->base);
1700
1701 ddata->clk = devm_clk_get(&pdev->dev, NULL);
1702 if (IS_ERR(ddata->clk))
1703 return dev_err_probe(&pdev->dev, PTR_ERR(ddata->clk), "Failed to get clk\n");
1704
1705 reset = devm_reset_control_get_optional(&pdev->dev, NULL);
1706 if (IS_ERR(reset))
1707 return dev_err_probe(&pdev->dev, PTR_ERR(reset), "Failed to get reset\n");
1708
1709 ret = clk_prepare_enable(ddata->clk);
1710 if (ret)
1711 return dev_err_probe(&pdev->dev, ret, "Failed to enable clk\n");
1712
1713 reset_control_reset(reset);
1714
1715 INIT_LIST_HEAD(&dma_dev->channels);
1716
1717 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1718 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1719 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1720 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1721 dma_dev->dev = &pdev->dev;
1722 /*
1723 * This controller supports up to 8-byte buswidth depending on the port used and the
1724 * channel, and can only access address at even boundaries, multiple of the buswidth.
1725 */
1726 dma_dev->copy_align = DMAENGINE_ALIGN_8_BYTES;
1727 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1728 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1729 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1730 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1731 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1732 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1733 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |
1734 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES);
1735 dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV) | BIT(DMA_MEM_TO_MEM);
1736
1737 dma_dev->descriptor_reuse = true;
1738 dma_dev->max_sg_burst = STM32_DMA3_MAX_SEG_SIZE;
1739 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1740 dma_dev->device_alloc_chan_resources = stm32_dma3_alloc_chan_resources;
1741 dma_dev->device_free_chan_resources = stm32_dma3_free_chan_resources;
1742 dma_dev->device_prep_dma_memcpy = stm32_dma3_prep_dma_memcpy;
1743 dma_dev->device_prep_slave_sg = stm32_dma3_prep_slave_sg;
1744 dma_dev->device_prep_dma_cyclic = stm32_dma3_prep_dma_cyclic;
1745 dma_dev->device_caps = stm32_dma3_caps;
1746 dma_dev->device_config = stm32_dma3_config;
1747 dma_dev->device_pause = stm32_dma3_pause;
1748 dma_dev->device_resume = stm32_dma3_resume;
1749 dma_dev->device_terminate_all = stm32_dma3_terminate_all;
1750 dma_dev->device_synchronize = stm32_dma3_synchronize;
1751 dma_dev->device_tx_status = stm32_dma3_tx_status;
1752 dma_dev->device_issue_pending = stm32_dma3_issue_pending;
1753
1754 /* if dma_channels is not modified, get it from hwcfgr1 */
1755 if (of_property_read_u32(np, "dma-channels", &ddata->dma_channels)) {
1756 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);
1757 ddata->dma_channels = FIELD_GET(G_NUM_CHANNELS, hwcfgr);
1758 }
1759
1760 /* if dma_requests is not modified, get it from hwcfgr2 */
1761 if (of_property_read_u32(np, "dma-requests", &ddata->dma_requests)) {
1762 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR2);
1763 ddata->dma_requests = FIELD_GET(G_MAX_REQ_ID, hwcfgr) + 1;
1764 }
1765
1766 /* G_MASTER_PORTS, G_M0_DATA_WIDTH_ENC, G_M1_DATA_WIDTH_ENC in HWCFGR1 */
1767 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR1);
1768 master_ports = FIELD_GET(G_MASTER_PORTS, hwcfgr);
1769
1770 ddata->ports_max_dw[0] = FIELD_GET(G_M0_DATA_WIDTH_ENC, hwcfgr);
1771 if (master_ports == AXI64 || master_ports == AHB32) /* Single master port */
1772 ddata->ports_max_dw[1] = DW_INVALID;
1773 else /* Dual master ports */
1774 ddata->ports_max_dw[1] = FIELD_GET(G_M1_DATA_WIDTH_ENC, hwcfgr);
1775
1776 /* axi_max_burst_len is optional, if not defined, use STM32_DMA3_MAX_BURST_LEN */
1777 ddata->axi_max_burst_len = STM32_DMA3_MAX_BURST_LEN;
1778 pdata = device_get_match_data(&pdev->dev);
1779 if (pdata && pdata->axi_max_burst_len) {
1780 ddata->axi_max_burst_len = min_t(u32, pdata->axi_max_burst_len,
1781 STM32_DMA3_MAX_BURST_LEN);
1782 dev_dbg(&pdev->dev, "Burst is limited to %u beats through AXI port\n",
1783 ddata->axi_max_burst_len);
1784 }
1785
1786 ddata->chans = devm_kcalloc(&pdev->dev, ddata->dma_channels, sizeof(*ddata->chans),
1787 GFP_KERNEL);
1788 if (!ddata->chans) {
1789 ret = -ENOMEM;
1790 goto err_clk_disable;
1791 }
1792
1793 chan_reserved = stm32_dma3_check_rif(ddata);
1794
1795 if (chan_reserved == GENMASK(ddata->dma_channels - 1, 0)) {
1796 ret = -ENODEV;
1797 dev_err_probe(&pdev->dev, ret, "No channel available, abort registration\n");
1798 goto err_clk_disable;
1799 }
1800
1801 /* G_FIFO_SIZE x=0..7 in HWCFGR3 and G_FIFO_SIZE x=8..15 in HWCFGR4 */
1802 hwcfgr = readl_relaxed(ddata->base + STM32_DMA3_HWCFGR3);
1803 hwcfgr |= ((u64)readl_relaxed(ddata->base + STM32_DMA3_HWCFGR4)) << 32;
1804
1805 for (i = 0; i < ddata->dma_channels; i++) {
1806 if (chan_reserved & BIT(i))
1807 continue;
1808
1809 chan = &ddata->chans[i];
1810 chan->id = i;
1811 chan->fifo_size = get_chan_hwcfg(i, G_FIFO_SIZE(i), hwcfgr);
1812 /* If chan->fifo_size > 0 then half of the fifo size, else no burst when no FIFO */
1813 chan->max_burst = (chan->fifo_size) ? (1 << (chan->fifo_size + 1)) / 2 : 0;
1814 }
1815
1816 ret = dmaenginem_async_device_register(dma_dev);
1817 if (ret)
1818 goto err_clk_disable;
1819
1820 for (i = 0; i < ddata->dma_channels; i++) {
1821 char name[12];
1822
1823 if (chan_reserved & BIT(i))
1824 continue;
1825
1826 chan = &ddata->chans[i];
1827 snprintf(name, sizeof(name), "dma%dchan%d", ddata->dma_dev.dev_id, chan->id);
1828
1829 chan->vchan.desc_free = stm32_dma3_chan_vdesc_free;
1830 vchan_init(&chan->vchan, dma_dev);
1831
1832 ret = dma_async_device_channel_register(&ddata->dma_dev, &chan->vchan.chan, name);
1833 if (ret) {
1834 dev_err_probe(&pdev->dev, ret, "Failed to register channel %s\n", name);
1835 goto err_clk_disable;
1836 }
1837
1838 ret = platform_get_irq(pdev, i);
1839 if (ret < 0)
1840 goto err_clk_disable;
1841 chan->irq = ret;
1842
1843 ret = devm_request_irq(&pdev->dev, chan->irq, stm32_dma3_chan_irq, 0,
1844 dev_name(chan2dev(chan)), chan);
1845 if (ret) {
1846 dev_err_probe(&pdev->dev, ret, "Failed to request channel %s IRQ\n",
1847 dev_name(chan2dev(chan)));
1848 goto err_clk_disable;
1849 }
1850 }
1851
1852 ret = of_dma_controller_register(np, stm32_dma3_of_xlate, ddata);
1853 if (ret) {
1854 dev_err_probe(&pdev->dev, ret, "Failed to register controller\n");
1855 goto err_clk_disable;
1856 }
1857
1858 verr = readl_relaxed(ddata->base + STM32_DMA3_VERR);
1859
1860 pm_runtime_set_active(&pdev->dev);
1861 pm_runtime_enable(&pdev->dev);
1862 pm_runtime_get_noresume(&pdev->dev);
1863 pm_runtime_put(&pdev->dev);
1864
1865 dev_info(&pdev->dev, "STM32 DMA3 registered rev:%lu.%lu\n",
1866 FIELD_GET(VERR_MAJREV, verr), FIELD_GET(VERR_MINREV, verr));
1867
1868 return 0;
1869
1870 err_clk_disable:
1871 clk_disable_unprepare(ddata->clk);
1872
1873 return ret;
1874 }
1875
1876 static void stm32_dma3_remove(struct platform_device *pdev)
1877 {
1878 pm_runtime_disable(&pdev->dev);
1879 }
1880
1881 static int stm32_dma3_runtime_suspend(struct device *dev)
1882 {
1883 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
1884
1885 clk_disable_unprepare(ddata->clk);
1886
1887 return 0;
1888 }
1889
1890 static int stm32_dma3_runtime_resume(struct device *dev)
1891 {
1892 struct stm32_dma3_ddata *ddata = dev_get_drvdata(dev);
1893 int ret;
1894
1895 ret = clk_prepare_enable(ddata->clk);
1896 if (ret)
1897 dev_err(dev, "Failed to enable clk: %d\n", ret);
1898
1899 return ret;
1900 }
1901
1902 static const struct dev_pm_ops stm32_dma3_pm_ops = {
1903 SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
1904 RUNTIME_PM_OPS(stm32_dma3_runtime_suspend, stm32_dma3_runtime_resume, NULL)
1905 };
1906
1907 static struct platform_driver stm32_dma3_driver = {
1908 .probe = stm32_dma3_probe,
1909 .remove = stm32_dma3_remove,
1910 .driver = {
1911 .name = "stm32-dma3",
1912 .of_match_table = stm32_dma3_of_match,
1913 .pm = pm_ptr(&stm32_dma3_pm_ops),
1914 },
1915 };
1916
1917 static int __init stm32_dma3_init(void)
1918 {
1919 return platform_driver_register(&stm32_dma3_driver);
1920 }
1921
1922 subsys_initcall(stm32_dma3_init);
1923
1924 MODULE_DESCRIPTION("STM32 DMA3 controller driver");
1925 MODULE_AUTHOR("Amelie Delaunay <amelie.delaunay@foss.st.com>");
1926 MODULE_LICENSE("GPL");
Kernel DRM miscellaneous fixes and cross-tree changes