?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/evalenti/linux...
[linux/fpc-iii.git] / drivers / dma / img-mdc-dma.c
bloba4c53be482cf3ce6ba639c510911e2060f07c616
1 /*
2 * IMG Multi-threaded DMA Controller (MDC)
3 *
4 * Copyright (C) 2009,2012,2013 Imagination Technologies Ltd.
5 * Copyright (C) 2014 Google, Inc.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 */
11
12 #include <linux/clk.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dmapool.h>
16 #include <linux/interrupt.h>
17 #include <linux/io.h>
18 #include <linux/irq.h>
19 #include <linux/kernel.h>
20 #include <linux/mfd/syscon.h>
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/of_device.h>
24 #include <linux/of_dma.h>
25 #include <linux/platform_device.h>
26 #include <linux/regmap.h>
27 #include <linux/slab.h>
28 #include <linux/spinlock.h>
29
30 #include "dmaengine.h"
31 #include "virt-dma.h"
32
33 #define MDC_MAX_DMA_CHANNELS 32
34
35 #define MDC_GENERAL_CONFIG 0x000
36 #define MDC_GENERAL_CONFIG_LIST_IEN BIT(31)
37 #define MDC_GENERAL_CONFIG_IEN BIT(29)
38 #define MDC_GENERAL_CONFIG_LEVEL_INT BIT(28)
39 #define MDC_GENERAL_CONFIG_INC_W BIT(12)
40 #define MDC_GENERAL_CONFIG_INC_R BIT(8)
41 #define MDC_GENERAL_CONFIG_PHYSICAL_W BIT(7)
42 #define MDC_GENERAL_CONFIG_WIDTH_W_SHIFT 4
43 #define MDC_GENERAL_CONFIG_WIDTH_W_MASK 0x7
44 #define MDC_GENERAL_CONFIG_PHYSICAL_R BIT(3)
45 #define MDC_GENERAL_CONFIG_WIDTH_R_SHIFT 0
46 #define MDC_GENERAL_CONFIG_WIDTH_R_MASK 0x7
47
48 #define MDC_READ_PORT_CONFIG 0x004
49 #define MDC_READ_PORT_CONFIG_STHREAD_SHIFT 28
50 #define MDC_READ_PORT_CONFIG_STHREAD_MASK 0xf
51 #define MDC_READ_PORT_CONFIG_RTHREAD_SHIFT 24
52 #define MDC_READ_PORT_CONFIG_RTHREAD_MASK 0xf
53 #define MDC_READ_PORT_CONFIG_WTHREAD_SHIFT 16
54 #define MDC_READ_PORT_CONFIG_WTHREAD_MASK 0xf
55 #define MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT 4
56 #define MDC_READ_PORT_CONFIG_BURST_SIZE_MASK 0xff
57 #define MDC_READ_PORT_CONFIG_DREQ_ENABLE BIT(1)
58
59 #define MDC_READ_ADDRESS 0x008
60
61 #define MDC_WRITE_ADDRESS 0x00c
62
63 #define MDC_TRANSFER_SIZE 0x010
64 #define MDC_TRANSFER_SIZE_MASK 0xffffff
65
66 #define MDC_LIST_NODE_ADDRESS 0x014
67
68 #define MDC_CMDS_PROCESSED 0x018
69 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT 16
70 #define MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK 0x3f
71 #define MDC_CMDS_PROCESSED_INT_ACTIVE BIT(8)
72 #define MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT 0
73 #define MDC_CMDS_PROCESSED_CMDS_DONE_MASK 0x3f
74
75 #define MDC_CONTROL_AND_STATUS 0x01c
76 #define MDC_CONTROL_AND_STATUS_CANCEL BIT(20)
77 #define MDC_CONTROL_AND_STATUS_LIST_EN BIT(4)
78 #define MDC_CONTROL_AND_STATUS_EN BIT(0)
79
80 #define MDC_ACTIVE_TRANSFER_SIZE 0x030
81
82 #define MDC_GLOBAL_CONFIG_A 0x900
83 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT 16
84 #define MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK 0xff
85 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT 8
86 #define MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK 0xff
87 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT 0
88 #define MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK 0xff
89
90 struct mdc_hw_list_desc {
91 u32 gen_conf;
92 u32 readport_conf;
93 u32 read_addr;
94 u32 write_addr;
95 u32 xfer_size;
96 u32 node_addr;
97 u32 cmds_done;
98 u32 ctrl_status;
99 /*
100 * Not part of the list descriptor, but instead used by the CPU to
101 * traverse the list.
102 */
103 struct mdc_hw_list_desc *next_desc;
104 };
105
106 struct mdc_tx_desc {
107 struct mdc_chan *chan;
108 struct virt_dma_desc vd;
109 dma_addr_t list_phys;
110 struct mdc_hw_list_desc *list;
111 bool cyclic;
112 bool cmd_loaded;
113 unsigned int list_len;
114 unsigned int list_period_len;
115 size_t list_xfer_size;
116 unsigned int list_cmds_done;
117 };
118
119 struct mdc_chan {
120 struct mdc_dma *mdma;
121 struct virt_dma_chan vc;
122 struct dma_slave_config config;
123 struct mdc_tx_desc *desc;
124 int irq;
125 unsigned int periph;
126 unsigned int thread;
127 unsigned int chan_nr;
128 };
129
130 struct mdc_dma_soc_data {
131 void (*enable_chan)(struct mdc_chan *mchan);
132 void (*disable_chan)(struct mdc_chan *mchan);
133 };
134
135 struct mdc_dma {
136 struct dma_device dma_dev;
137 void __iomem *regs;
138 struct clk *clk;
139 struct dma_pool *desc_pool;
140 struct regmap *periph_regs;
141 spinlock_t lock;
142 unsigned int nr_threads;
143 unsigned int nr_channels;
144 unsigned int bus_width;
145 unsigned int max_burst_mult;
146 unsigned int max_xfer_size;
147 const struct mdc_dma_soc_data *soc;
148 struct mdc_chan channels[MDC_MAX_DMA_CHANNELS];
149 };
150
151 static inline u32 mdc_readl(struct mdc_dma *mdma, u32 reg)
152 {
153 return readl(mdma->regs + reg);
154 }
155
156 static inline void mdc_writel(struct mdc_dma *mdma, u32 val, u32 reg)
157 {
158 writel(val, mdma->regs + reg);
159 }
160
161 static inline u32 mdc_chan_readl(struct mdc_chan *mchan, u32 reg)
162 {
163 return mdc_readl(mchan->mdma, mchan->chan_nr * 0x040 + reg);
164 }
165
166 static inline void mdc_chan_writel(struct mdc_chan *mchan, u32 val, u32 reg)
167 {
168 mdc_writel(mchan->mdma, val, mchan->chan_nr * 0x040 + reg);
169 }
170
171 static inline struct mdc_chan *to_mdc_chan(struct dma_chan *c)
172 {
173 return container_of(to_virt_chan(c), struct mdc_chan, vc);
174 }
175
176 static inline struct mdc_tx_desc *to_mdc_desc(struct dma_async_tx_descriptor *t)
177 {
178 struct virt_dma_desc *vdesc = container_of(t, struct virt_dma_desc, tx);
179
180 return container_of(vdesc, struct mdc_tx_desc, vd);
181 }
182
183 static inline struct device *mdma2dev(struct mdc_dma *mdma)
184 {
185 return mdma->dma_dev.dev;
186 }
187
188 static inline unsigned int to_mdc_width(unsigned int bytes)
189 {
190 return ffs(bytes) - 1;
191 }
192
193 static inline void mdc_set_read_width(struct mdc_hw_list_desc *ldesc,
194 unsigned int bytes)
195 {
196 ldesc->gen_conf |= to_mdc_width(bytes) <<
197 MDC_GENERAL_CONFIG_WIDTH_R_SHIFT;
198 }
199
200 static inline void mdc_set_write_width(struct mdc_hw_list_desc *ldesc,
201 unsigned int bytes)
202 {
203 ldesc->gen_conf |= to_mdc_width(bytes) <<
204 MDC_GENERAL_CONFIG_WIDTH_W_SHIFT;
205 }
206
207 static void mdc_list_desc_config(struct mdc_chan *mchan,
208 struct mdc_hw_list_desc *ldesc,
209 enum dma_transfer_direction dir,
210 dma_addr_t src, dma_addr_t dst, size_t len)
211 {
212 struct mdc_dma *mdma = mchan->mdma;
213 unsigned int max_burst, burst_size;
214
215 ldesc->gen_conf = MDC_GENERAL_CONFIG_IEN | MDC_GENERAL_CONFIG_LIST_IEN |
216 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
217 MDC_GENERAL_CONFIG_PHYSICAL_R;
218 ldesc->readport_conf =
219 (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
220 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
221 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
222 ldesc->read_addr = src;
223 ldesc->write_addr = dst;
224 ldesc->xfer_size = len - 1;
225 ldesc->node_addr = 0;
226 ldesc->cmds_done = 0;
227 ldesc->ctrl_status = MDC_CONTROL_AND_STATUS_LIST_EN |
228 MDC_CONTROL_AND_STATUS_EN;
229 ldesc->next_desc = NULL;
230
231 if (IS_ALIGNED(dst, mdma->bus_width) &&
232 IS_ALIGNED(src, mdma->bus_width))
233 max_burst = mdma->bus_width * mdma->max_burst_mult;
234 else
235 max_burst = mdma->bus_width * (mdma->max_burst_mult - 1);
236
237 if (dir == DMA_MEM_TO_DEV) {
238 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R;
239 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
240 mdc_set_read_width(ldesc, mdma->bus_width);
241 mdc_set_write_width(ldesc, mchan->config.dst_addr_width);
242 burst_size = min(max_burst, mchan->config.dst_maxburst *
243 mchan->config.dst_addr_width);
244 } else if (dir == DMA_DEV_TO_MEM) {
245 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_W;
246 ldesc->readport_conf |= MDC_READ_PORT_CONFIG_DREQ_ENABLE;
247 mdc_set_read_width(ldesc, mchan->config.src_addr_width);
248 mdc_set_write_width(ldesc, mdma->bus_width);
249 burst_size = min(max_burst, mchan->config.src_maxburst *
250 mchan->config.src_addr_width);
251 } else {
252 ldesc->gen_conf |= MDC_GENERAL_CONFIG_INC_R |
253 MDC_GENERAL_CONFIG_INC_W;
254 mdc_set_read_width(ldesc, mdma->bus_width);
255 mdc_set_write_width(ldesc, mdma->bus_width);
256 burst_size = max_burst;
257 }
258 ldesc->readport_conf |= (burst_size - 1) <<
259 MDC_READ_PORT_CONFIG_BURST_SIZE_SHIFT;
260 }
261
262 static void mdc_list_desc_free(struct mdc_tx_desc *mdesc)
263 {
264 struct mdc_dma *mdma = mdesc->chan->mdma;
265 struct mdc_hw_list_desc *curr, *next;
266 dma_addr_t curr_phys, next_phys;
267
268 curr = mdesc->list;
269 curr_phys = mdesc->list_phys;
270 while (curr) {
271 next = curr->next_desc;
272 next_phys = curr->node_addr;
273 dma_pool_free(mdma->desc_pool, curr, curr_phys);
274 curr = next;
275 curr_phys = next_phys;
276 }
277 }
278
279 static void mdc_desc_free(struct virt_dma_desc *vd)
280 {
281 struct mdc_tx_desc *mdesc = to_mdc_desc(&vd->tx);
282
283 mdc_list_desc_free(mdesc);
284 kfree(mdesc);
285 }
286
287 static struct dma_async_tx_descriptor *mdc_prep_dma_memcpy(
288 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, size_t len,
289 unsigned long flags)
290 {
291 struct mdc_chan *mchan = to_mdc_chan(chan);
292 struct mdc_dma *mdma = mchan->mdma;
293 struct mdc_tx_desc *mdesc;
294 struct mdc_hw_list_desc *curr, *prev = NULL;
295 dma_addr_t curr_phys, prev_phys;
296
297 if (!len)
298 return NULL;
299
300 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
301 if (!mdesc)
302 return NULL;
303 mdesc->chan = mchan;
304 mdesc->list_xfer_size = len;
305
306 while (len > 0) {
307 size_t xfer_size;
308
309 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT, &curr_phys);
310 if (!curr)
311 goto free_desc;
312
313 if (prev) {
314 prev->node_addr = curr_phys;
315 prev->next_desc = curr;
316 } else {
317 mdesc->list_phys = curr_phys;
318 mdesc->list = curr;
319 }
320
321 xfer_size = min_t(size_t, mdma->max_xfer_size, len);
322
323 mdc_list_desc_config(mchan, curr, DMA_MEM_TO_MEM, src, dest,
324 xfer_size);
325
326 prev = curr;
327 prev_phys = curr_phys;
328
329 mdesc->list_len++;
330 src += xfer_size;
331 dest += xfer_size;
332 len -= xfer_size;
333 }
334
335 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
336
337 free_desc:
338 mdc_desc_free(&mdesc->vd);
339
340 return NULL;
341 }
342
343 static int mdc_check_slave_width(struct mdc_chan *mchan,
344 enum dma_transfer_direction dir)
345 {
346 enum dma_slave_buswidth width;
347
348 if (dir == DMA_MEM_TO_DEV)
349 width = mchan->config.dst_addr_width;
350 else
351 width = mchan->config.src_addr_width;
352
353 switch (width) {
354 case DMA_SLAVE_BUSWIDTH_1_BYTE:
355 case DMA_SLAVE_BUSWIDTH_2_BYTES:
356 case DMA_SLAVE_BUSWIDTH_4_BYTES:
357 case DMA_SLAVE_BUSWIDTH_8_BYTES:
358 break;
359 default:
360 return -EINVAL;
361 }
362
363 if (width > mchan->mdma->bus_width)
364 return -EINVAL;
365
366 return 0;
367 }
368
369 static struct dma_async_tx_descriptor *mdc_prep_dma_cyclic(
370 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
371 size_t period_len, enum dma_transfer_direction dir,
372 unsigned long flags)
373 {
374 struct mdc_chan *mchan = to_mdc_chan(chan);
375 struct mdc_dma *mdma = mchan->mdma;
376 struct mdc_tx_desc *mdesc;
377 struct mdc_hw_list_desc *curr, *prev = NULL;
378 dma_addr_t curr_phys, prev_phys;
379
380 if (!buf_len && !period_len)
381 return NULL;
382
383 if (!is_slave_direction(dir))
384 return NULL;
385
386 if (mdc_check_slave_width(mchan, dir) < 0)
387 return NULL;
388
389 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
390 if (!mdesc)
391 return NULL;
392 mdesc->chan = mchan;
393 mdesc->cyclic = true;
394 mdesc->list_xfer_size = buf_len;
395 mdesc->list_period_len = DIV_ROUND_UP(period_len,
396 mdma->max_xfer_size);
397
398 while (buf_len > 0) {
399 size_t remainder = min(period_len, buf_len);
400
401 while (remainder > 0) {
402 size_t xfer_size;
403
404 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
405 &curr_phys);
406 if (!curr)
407 goto free_desc;
408
409 if (!prev) {
410 mdesc->list_phys = curr_phys;
411 mdesc->list = curr;
412 } else {
413 prev->node_addr = curr_phys;
414 prev->next_desc = curr;
415 }
416
417 xfer_size = min_t(size_t, mdma->max_xfer_size,
418 remainder);
419
420 if (dir == DMA_MEM_TO_DEV) {
421 mdc_list_desc_config(mchan, curr, dir,
422 buf_addr,
423 mchan->config.dst_addr,
424 xfer_size);
425 } else {
426 mdc_list_desc_config(mchan, curr, dir,
427 mchan->config.src_addr,
428 buf_addr,
429 xfer_size);
430 }
431
432 prev = curr;
433 prev_phys = curr_phys;
434
435 mdesc->list_len++;
436 buf_addr += xfer_size;
437 buf_len -= xfer_size;
438 remainder -= xfer_size;
439 }
440 }
441 prev->node_addr = mdesc->list_phys;
442
443 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
444
445 free_desc:
446 mdc_desc_free(&mdesc->vd);
447
448 return NULL;
449 }
450
451 static struct dma_async_tx_descriptor *mdc_prep_slave_sg(
452 struct dma_chan *chan, struct scatterlist *sgl,
453 unsigned int sg_len, enum dma_transfer_direction dir,
454 unsigned long flags, void *context)
455 {
456 struct mdc_chan *mchan = to_mdc_chan(chan);
457 struct mdc_dma *mdma = mchan->mdma;
458 struct mdc_tx_desc *mdesc;
459 struct scatterlist *sg;
460 struct mdc_hw_list_desc *curr, *prev = NULL;
461 dma_addr_t curr_phys, prev_phys;
462 unsigned int i;
463
464 if (!sgl)
465 return NULL;
466
467 if (!is_slave_direction(dir))
468 return NULL;
469
470 if (mdc_check_slave_width(mchan, dir) < 0)
471 return NULL;
472
473 mdesc = kzalloc(sizeof(*mdesc), GFP_NOWAIT);
474 if (!mdesc)
475 return NULL;
476 mdesc->chan = mchan;
477
478 for_each_sg(sgl, sg, sg_len, i) {
479 dma_addr_t buf = sg_dma_address(sg);
480 size_t buf_len = sg_dma_len(sg);
481
482 while (buf_len > 0) {
483 size_t xfer_size;
484
485 curr = dma_pool_alloc(mdma->desc_pool, GFP_NOWAIT,
486 &curr_phys);
487 if (!curr)
488 goto free_desc;
489
490 if (!prev) {
491 mdesc->list_phys = curr_phys;
492 mdesc->list = curr;
493 } else {
494 prev->node_addr = curr_phys;
495 prev->next_desc = curr;
496 }
497
498 xfer_size = min_t(size_t, mdma->max_xfer_size,
499 buf_len);
500
501 if (dir == DMA_MEM_TO_DEV) {
502 mdc_list_desc_config(mchan, curr, dir, buf,
503 mchan->config.dst_addr,
504 xfer_size);
505 } else {
506 mdc_list_desc_config(mchan, curr, dir,
507 mchan->config.src_addr,
508 buf, xfer_size);
509 }
510
511 prev = curr;
512 prev_phys = curr_phys;
513
514 mdesc->list_len++;
515 mdesc->list_xfer_size += xfer_size;
516 buf += xfer_size;
517 buf_len -= xfer_size;
518 }
519 }
520
521 return vchan_tx_prep(&mchan->vc, &mdesc->vd, flags);
522
523 free_desc:
524 mdc_desc_free(&mdesc->vd);
525
526 return NULL;
527 }
528
529 static void mdc_issue_desc(struct mdc_chan *mchan)
530 {
531 struct mdc_dma *mdma = mchan->mdma;
532 struct virt_dma_desc *vd;
533 struct mdc_tx_desc *mdesc;
534 u32 val;
535
536 vd = vchan_next_desc(&mchan->vc);
537 if (!vd)
538 return;
539
540 list_del(&vd->node);
541
542 mdesc = to_mdc_desc(&vd->tx);
543 mchan->desc = mdesc;
544
545 dev_dbg(mdma2dev(mdma), "Issuing descriptor on channel %d\n",
546 mchan->chan_nr);
547
548 mdma->soc->enable_chan(mchan);
549
550 val = mdc_chan_readl(mchan, MDC_GENERAL_CONFIG);
551 val |= MDC_GENERAL_CONFIG_LIST_IEN | MDC_GENERAL_CONFIG_IEN |
552 MDC_GENERAL_CONFIG_LEVEL_INT | MDC_GENERAL_CONFIG_PHYSICAL_W |
553 MDC_GENERAL_CONFIG_PHYSICAL_R;
554 mdc_chan_writel(mchan, val, MDC_GENERAL_CONFIG);
555 val = (mchan->thread << MDC_READ_PORT_CONFIG_STHREAD_SHIFT) |
556 (mchan->thread << MDC_READ_PORT_CONFIG_RTHREAD_SHIFT) |
557 (mchan->thread << MDC_READ_PORT_CONFIG_WTHREAD_SHIFT);
558 mdc_chan_writel(mchan, val, MDC_READ_PORT_CONFIG);
559 mdc_chan_writel(mchan, mdesc->list_phys, MDC_LIST_NODE_ADDRESS);
560 val = mdc_chan_readl(mchan, MDC_CONTROL_AND_STATUS);
561 val |= MDC_CONTROL_AND_STATUS_LIST_EN;
562 mdc_chan_writel(mchan, val, MDC_CONTROL_AND_STATUS);
563 }
564
565 static void mdc_issue_pending(struct dma_chan *chan)
566 {
567 struct mdc_chan *mchan = to_mdc_chan(chan);
568 unsigned long flags;
569
570 spin_lock_irqsave(&mchan->vc.lock, flags);
571 if (vchan_issue_pending(&mchan->vc) && !mchan->desc)
572 mdc_issue_desc(mchan);
573 spin_unlock_irqrestore(&mchan->vc.lock, flags);
574 }
575
576 static enum dma_status mdc_tx_status(struct dma_chan *chan,
577 dma_cookie_t cookie, struct dma_tx_state *txstate)
578 {
579 struct mdc_chan *mchan = to_mdc_chan(chan);
580 struct mdc_tx_desc *mdesc;
581 struct virt_dma_desc *vd;
582 unsigned long flags;
583 size_t bytes = 0;
584 int ret;
585
586 ret = dma_cookie_status(chan, cookie, txstate);
587 if (ret == DMA_COMPLETE)
588 return ret;
589
590 if (!txstate)
591 return ret;
592
593 spin_lock_irqsave(&mchan->vc.lock, flags);
594 vd = vchan_find_desc(&mchan->vc, cookie);
595 if (vd) {
596 mdesc = to_mdc_desc(&vd->tx);
597 bytes = mdesc->list_xfer_size;
598 } else if (mchan->desc && mchan->desc->vd.tx.cookie == cookie) {
599 struct mdc_hw_list_desc *ldesc;
600 u32 val1, val2, done, processed, residue;
601 int i, cmds;
602
603 mdesc = mchan->desc;
604
605 /*
606 * Determine the number of commands that haven't been
607 * processed (handled by the IRQ handler) yet.
608 */
609 do {
610 val1 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
611 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
612 residue = mdc_chan_readl(mchan,
613 MDC_ACTIVE_TRANSFER_SIZE);
614 val2 = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED) &
615 ~MDC_CMDS_PROCESSED_INT_ACTIVE;
616 } while (val1 != val2);
617
618 done = (val1 >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
619 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
620 processed = (val1 >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
621 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
622 cmds = (done - processed) %
623 (MDC_CMDS_PROCESSED_CMDS_DONE_MASK + 1);
624
625 /*
626 * If the command loaded event hasn't been processed yet, then
627 * the difference above includes an extra command.
628 */
629 if (!mdesc->cmd_loaded)
630 cmds--;
631 else
632 cmds += mdesc->list_cmds_done;
633
634 bytes = mdesc->list_xfer_size;
635 ldesc = mdesc->list;
636 for (i = 0; i < cmds; i++) {
637 bytes -= ldesc->xfer_size + 1;
638 ldesc = ldesc->next_desc;
639 }
640 if (ldesc) {
641 if (residue != MDC_TRANSFER_SIZE_MASK)
642 bytes -= ldesc->xfer_size - residue;
643 else
644 bytes -= ldesc->xfer_size + 1;
645 }
646 }
647 spin_unlock_irqrestore(&mchan->vc.lock, flags);
648
649 dma_set_residue(txstate, bytes);
650
651 return ret;
652 }
653
654 static unsigned int mdc_get_new_events(struct mdc_chan *mchan)
655 {
656 u32 val, processed, done1, done2;
657 unsigned int ret;
658
659 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
660 processed = (val >> MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) &
661 MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK;
662 /*
663 * CMDS_DONE may have incremented between reading CMDS_PROCESSED
664 * and clearing INT_ACTIVE. Re-read CMDS_PROCESSED to ensure we
665 * didn't miss a command completion.
666 */
667 do {
668 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
669
670 done1 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
671 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
672
673 val &= ~((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK <<
674 MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT) |
675 MDC_CMDS_PROCESSED_INT_ACTIVE);
676
677 val |= done1 << MDC_CMDS_PROCESSED_CMDS_PROCESSED_SHIFT;
678
679 mdc_chan_writel(mchan, val, MDC_CMDS_PROCESSED);
680
681 val = mdc_chan_readl(mchan, MDC_CMDS_PROCESSED);
682
683 done2 = (val >> MDC_CMDS_PROCESSED_CMDS_DONE_SHIFT) &
684 MDC_CMDS_PROCESSED_CMDS_DONE_MASK;
685 } while (done1 != done2);
686
687 if (done1 >= processed)
688 ret = done1 - processed;
689 else
690 ret = ((MDC_CMDS_PROCESSED_CMDS_PROCESSED_MASK + 1) -
691 processed) + done1;
692
693 return ret;
694 }
695
696 static int mdc_terminate_all(struct dma_chan *chan)
697 {
698 struct mdc_chan *mchan = to_mdc_chan(chan);
699 struct mdc_tx_desc *mdesc;
700 unsigned long flags;
701 LIST_HEAD(head);
702
703 spin_lock_irqsave(&mchan->vc.lock, flags);
704
705 mdc_chan_writel(mchan, MDC_CONTROL_AND_STATUS_CANCEL,
706 MDC_CONTROL_AND_STATUS);
707
708 mdesc = mchan->desc;
709 mchan->desc = NULL;
710 vchan_get_all_descriptors(&mchan->vc, &head);
711
712 mdc_get_new_events(mchan);
713
714 spin_unlock_irqrestore(&mchan->vc.lock, flags);
715
716 if (mdesc)
717 mdc_desc_free(&mdesc->vd);
718 vchan_dma_desc_free_list(&mchan->vc, &head);
719
720 return 0;
721 }
722
723 static int mdc_slave_config(struct dma_chan *chan,
724 struct dma_slave_config *config)
725 {
726 struct mdc_chan *mchan = to_mdc_chan(chan);
727 unsigned long flags;
728
729 spin_lock_irqsave(&mchan->vc.lock, flags);
730 mchan->config = *config;
731 spin_unlock_irqrestore(&mchan->vc.lock, flags);
732
733 return 0;
734 }
735
736 static void mdc_free_chan_resources(struct dma_chan *chan)
737 {
738 struct mdc_chan *mchan = to_mdc_chan(chan);
739 struct mdc_dma *mdma = mchan->mdma;
740
741 mdc_terminate_all(chan);
742
743 mdma->soc->disable_chan(mchan);
744 }
745
746 static irqreturn_t mdc_chan_irq(int irq, void *dev_id)
747 {
748 struct mdc_chan *mchan = (struct mdc_chan *)dev_id;
749 struct mdc_tx_desc *mdesc;
750 unsigned int i, new_events;
751
752 spin_lock(&mchan->vc.lock);
753
754 dev_dbg(mdma2dev(mchan->mdma), "IRQ on channel %d\n", mchan->chan_nr);
755
756 new_events = mdc_get_new_events(mchan);
757
758 if (!new_events)
759 goto out;
760
761 mdesc = mchan->desc;
762 if (!mdesc) {
763 dev_warn(mdma2dev(mchan->mdma),
764 "IRQ with no active descriptor on channel %d\n",
765 mchan->chan_nr);
766 goto out;
767 }
768
769 for (i = 0; i < new_events; i++) {
770 /*
771 * The first interrupt in a transfer indicates that the
772 * command list has been loaded, not that a command has
773 * been completed.
774 */
775 if (!mdesc->cmd_loaded) {
776 mdesc->cmd_loaded = true;
777 continue;
778 }
779
780 mdesc->list_cmds_done++;
781 if (mdesc->cyclic) {
782 mdesc->list_cmds_done %= mdesc->list_len;
783 if (mdesc->list_cmds_done % mdesc->list_period_len == 0)
784 vchan_cyclic_callback(&mdesc->vd);
785 } else if (mdesc->list_cmds_done == mdesc->list_len) {
786 mchan->desc = NULL;
787 vchan_cookie_complete(&mdesc->vd);
788 mdc_issue_desc(mchan);
789 break;
790 }
791 }
792 out:
793 spin_unlock(&mchan->vc.lock);
794
795 return IRQ_HANDLED;
796 }
797
798 static struct dma_chan *mdc_of_xlate(struct of_phandle_args *dma_spec,
799 struct of_dma *ofdma)
800 {
801 struct mdc_dma *mdma = ofdma->of_dma_data;
802 struct dma_chan *chan;
803
804 if (dma_spec->args_count != 3)
805 return NULL;
806
807 list_for_each_entry(chan, &mdma->dma_dev.channels, device_node) {
808 struct mdc_chan *mchan = to_mdc_chan(chan);
809
810 if (!(dma_spec->args[1] & BIT(mchan->chan_nr)))
811 continue;
812 if (dma_get_slave_channel(chan)) {
813 mchan->periph = dma_spec->args[0];
814 mchan->thread = dma_spec->args[2];
815 return chan;
816 }
817 }
818
819 return NULL;
820 }
821
822 #define PISTACHIO_CR_PERIPH_DMA_ROUTE(ch) (0x120 + 0x4 * ((ch) / 4))
823 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(ch) (8 * ((ch) % 4))
824 #define PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK 0x3f
825
826 static void pistachio_mdc_enable_chan(struct mdc_chan *mchan)
827 {
828 struct mdc_dma *mdma = mchan->mdma;
829
830 regmap_update_bits(mdma->periph_regs,
831 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
832 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
833 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
834 mchan->periph <<
835 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr));
836 }
837
838 static void pistachio_mdc_disable_chan(struct mdc_chan *mchan)
839 {
840 struct mdc_dma *mdma = mchan->mdma;
841
842 regmap_update_bits(mdma->periph_regs,
843 PISTACHIO_CR_PERIPH_DMA_ROUTE(mchan->chan_nr),
844 PISTACHIO_CR_PERIPH_DMA_ROUTE_MASK <<
845 PISTACHIO_CR_PERIPH_DMA_ROUTE_SHIFT(mchan->chan_nr),
846 0);
847 }
848
849 static const struct mdc_dma_soc_data pistachio_mdc_data = {
850 .enable_chan = pistachio_mdc_enable_chan,
851 .disable_chan = pistachio_mdc_disable_chan,
852 };
853
854 static const struct of_device_id mdc_dma_of_match[] = {
855 { .compatible = "img,pistachio-mdc-dma", .data = &pistachio_mdc_data, },
856 { },
857 };
858 MODULE_DEVICE_TABLE(of, mdc_dma_of_match);
859
860 static int mdc_dma_probe(struct platform_device *pdev)
861 {
862 struct mdc_dma *mdma;
863 struct resource *res;
864 const struct of_device_id *match;
865 unsigned int i;
866 u32 val;
867 int ret;
868
869 mdma = devm_kzalloc(&pdev->dev, sizeof(*mdma), GFP_KERNEL);
870 if (!mdma)
871 return -ENOMEM;
872 platform_set_drvdata(pdev, mdma);
873
874 match = of_match_device(mdc_dma_of_match, &pdev->dev);
875 mdma->soc = match->data;
876
877 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
878 mdma->regs = devm_ioremap_resource(&pdev->dev, res);
879 if (IS_ERR(mdma->regs))
880 return PTR_ERR(mdma->regs);
881
882 mdma->periph_regs = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
883 "img,cr-periph");
884 if (IS_ERR(mdma->periph_regs))
885 return PTR_ERR(mdma->periph_regs);
886
887 mdma->clk = devm_clk_get(&pdev->dev, "sys");
888 if (IS_ERR(mdma->clk))
889 return PTR_ERR(mdma->clk);
890
891 ret = clk_prepare_enable(mdma->clk);
892 if (ret)
893 return ret;
894
895 dma_cap_zero(mdma->dma_dev.cap_mask);
896 dma_cap_set(DMA_SLAVE, mdma->dma_dev.cap_mask);
897 dma_cap_set(DMA_PRIVATE, mdma->dma_dev.cap_mask);
898 dma_cap_set(DMA_CYCLIC, mdma->dma_dev.cap_mask);
899 dma_cap_set(DMA_MEMCPY, mdma->dma_dev.cap_mask);
900
901 val = mdc_readl(mdma, MDC_GLOBAL_CONFIG_A);
902 mdma->nr_channels = (val >> MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_SHIFT) &
903 MDC_GLOBAL_CONFIG_A_DMA_CONTEXTS_MASK;
904 mdma->nr_threads =
905 1 << ((val >> MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_SHIFT) &
906 MDC_GLOBAL_CONFIG_A_THREAD_ID_WIDTH_MASK);
907 mdma->bus_width =
908 (1 << ((val >> MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_SHIFT) &
909 MDC_GLOBAL_CONFIG_A_SYS_DAT_WIDTH_MASK)) / 8;
910 /*
911 * Although transfer sizes of up to MDC_TRANSFER_SIZE_MASK + 1 bytes
912 * are supported, this makes it possible for the value reported in
913 * MDC_ACTIVE_TRANSFER_SIZE to be ambiguous - an active transfer size
914 * of MDC_TRANSFER_SIZE_MASK may indicate either that 0 bytes or
915 * MDC_TRANSFER_SIZE_MASK + 1 bytes are remaining. To eliminate this
916 * ambiguity, restrict transfer sizes to one bus-width less than the
917 * actual maximum.
918 */
919 mdma->max_xfer_size = MDC_TRANSFER_SIZE_MASK + 1 - mdma->bus_width;
920
921 of_property_read_u32(pdev->dev.of_node, "dma-channels",
922 &mdma->nr_channels);
923 ret = of_property_read_u32(pdev->dev.of_node,
924 "img,max-burst-multiplier",
925 &mdma->max_burst_mult);
926 if (ret)
927 goto disable_clk;
928
929 mdma->dma_dev.dev = &pdev->dev;
930 mdma->dma_dev.device_prep_slave_sg = mdc_prep_slave_sg;
931 mdma->dma_dev.device_prep_dma_cyclic = mdc_prep_dma_cyclic;
932 mdma->dma_dev.device_prep_dma_memcpy = mdc_prep_dma_memcpy;
933 mdma->dma_dev.device_free_chan_resources = mdc_free_chan_resources;
934 mdma->dma_dev.device_tx_status = mdc_tx_status;
935 mdma->dma_dev.device_issue_pending = mdc_issue_pending;
936 mdma->dma_dev.device_terminate_all = mdc_terminate_all;
937 mdma->dma_dev.device_config = mdc_slave_config;
938
939 mdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
940 mdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
941 for (i = 1; i <= mdma->bus_width; i <<= 1) {
942 mdma->dma_dev.src_addr_widths |= BIT(i);
943 mdma->dma_dev.dst_addr_widths |= BIT(i);
944 }
945
946 INIT_LIST_HEAD(&mdma->dma_dev.channels);
947 for (i = 0; i < mdma->nr_channels; i++) {
948 struct mdc_chan *mchan = &mdma->channels[i];
949
950 mchan->mdma = mdma;
951 mchan->chan_nr = i;
952 mchan->irq = platform_get_irq(pdev, i);
953 if (mchan->irq < 0) {
954 ret = mchan->irq;
955 goto disable_clk;
956 }
957 ret = devm_request_irq(&pdev->dev, mchan->irq, mdc_chan_irq,
958 IRQ_TYPE_LEVEL_HIGH,
959 dev_name(&pdev->dev), mchan);
960 if (ret < 0)
961 goto disable_clk;
962
963 mchan->vc.desc_free = mdc_desc_free;
964 vchan_init(&mchan->vc, &mdma->dma_dev);
965 }
966
967 mdma->desc_pool = dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
968 sizeof(struct mdc_hw_list_desc),
969 4, 0);
970 if (!mdma->desc_pool) {
971 ret = -ENOMEM;
972 goto disable_clk;
973 }
974
975 ret = dma_async_device_register(&mdma->dma_dev);
976 if (ret)
977 goto disable_clk;
978
979 ret = of_dma_controller_register(pdev->dev.of_node, mdc_of_xlate, mdma);
980 if (ret)
981 goto unregister;
982
983 dev_info(&pdev->dev, "MDC with %u channels and %u threads\n",
984 mdma->nr_channels, mdma->nr_threads);
985
986 return 0;
987
988 unregister:
989 dma_async_device_unregister(&mdma->dma_dev);
990 disable_clk:
991 clk_disable_unprepare(mdma->clk);
992 return ret;
993 }
994
995 static int mdc_dma_remove(struct platform_device *pdev)
996 {
997 struct mdc_dma *mdma = platform_get_drvdata(pdev);
998 struct mdc_chan *mchan, *next;
999
1000 of_dma_controller_free(pdev->dev.of_node);
1001 dma_async_device_unregister(&mdma->dma_dev);
1002
1003 list_for_each_entry_safe(mchan, next, &mdma->dma_dev.channels,
1004 vc.chan.device_node) {
1005 list_del(&mchan->vc.chan.device_node);
1006
1007 devm_free_irq(&pdev->dev, mchan->irq, mchan);
1008
1009 tasklet_kill(&mchan->vc.task);
1010 }
1011
1012 clk_disable_unprepare(mdma->clk);
1013
1014 return 0;
1015 }
1016
1017 static struct platform_driver mdc_dma_driver = {
1018 .driver = {
1019 .name = "img-mdc-dma",
1020 .of_match_table = of_match_ptr(mdc_dma_of_match),
1021 },
1022 .probe = mdc_dma_probe,
1023 .remove = mdc_dma_remove,
1024 };
1025 module_platform_driver(mdc_dma_driver);
1026
1027 MODULE_DESCRIPTION("IMG Multi-threaded DMA Controller (MDC) driver");
1028 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
1029 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support