2 * Core driver for the Intel integrated DMA 64-bit
4 * Copyright (C) 2015 Intel Corporation
5 * Author: Andy Shevchenko <andriy.shevchenko@linux.intel.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/bitops.h>
13 #include <linux/delay.h>
14 #include <linux/dmaengine.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dmapool.h>
17 #include <linux/init.h>
18 #include <linux/module.h>
19 #include <linux/platform_device.h>
20 #include <linux/slab.h>
24 /* Platform driver name */
25 #define DRV_NAME "idma64"
27 /* For now we support only two channels */
28 #define IDMA64_NR_CHAN 2
30 /* ---------------------------------------------------------------------- */
32 static struct device
*chan2dev(struct dma_chan
*chan
)
34 return &chan
->dev
->device
;
37 /* ---------------------------------------------------------------------- */
39 static void idma64_off(struct idma64
*idma64
)
41 unsigned short count
= 100;
43 dma_writel(idma64
, CFG
, 0);
45 channel_clear_bit(idma64
, MASK(XFER
), idma64
->all_chan_mask
);
46 channel_clear_bit(idma64
, MASK(BLOCK
), idma64
->all_chan_mask
);
47 channel_clear_bit(idma64
, MASK(SRC_TRAN
), idma64
->all_chan_mask
);
48 channel_clear_bit(idma64
, MASK(DST_TRAN
), idma64
->all_chan_mask
);
49 channel_clear_bit(idma64
, MASK(ERROR
), idma64
->all_chan_mask
);
53 } while (dma_readl(idma64
, CFG
) & IDMA64_CFG_DMA_EN
&& --count
);
56 static void idma64_on(struct idma64
*idma64
)
58 dma_writel(idma64
, CFG
, IDMA64_CFG_DMA_EN
);
61 /* ---------------------------------------------------------------------- */
63 static void idma64_chan_init(struct idma64
*idma64
, struct idma64_chan
*idma64c
)
65 u32 cfghi
= IDMA64C_CFGH_SRC_PER(1) | IDMA64C_CFGH_DST_PER(0);
68 /* Set default burst alignment */
69 cfglo
|= IDMA64C_CFGL_DST_BURST_ALIGN
| IDMA64C_CFGL_SRC_BURST_ALIGN
;
71 channel_writel(idma64c
, CFG_LO
, cfglo
);
72 channel_writel(idma64c
, CFG_HI
, cfghi
);
74 /* Enable interrupts */
75 channel_set_bit(idma64
, MASK(XFER
), idma64c
->mask
);
76 channel_set_bit(idma64
, MASK(ERROR
), idma64c
->mask
);
79 * Enforce the controller to be turned on.
81 * The iDMA is turned off in ->probe() and looses context during system
82 * suspend / resume cycle. That's why we have to enable it each time we
88 static void idma64_chan_stop(struct idma64
*idma64
, struct idma64_chan
*idma64c
)
90 channel_clear_bit(idma64
, CH_EN
, idma64c
->mask
);
93 static void idma64_chan_start(struct idma64
*idma64
, struct idma64_chan
*idma64c
)
95 struct idma64_desc
*desc
= idma64c
->desc
;
96 struct idma64_hw_desc
*hw
= &desc
->hw
[0];
98 channel_writeq(idma64c
, SAR
, 0);
99 channel_writeq(idma64c
, DAR
, 0);
101 channel_writel(idma64c
, CTL_HI
, IDMA64C_CTLH_BLOCK_TS(~0UL));
102 channel_writel(idma64c
, CTL_LO
, IDMA64C_CTLL_LLP_S_EN
| IDMA64C_CTLL_LLP_D_EN
);
104 channel_writeq(idma64c
, LLP
, hw
->llp
);
106 channel_set_bit(idma64
, CH_EN
, idma64c
->mask
);
109 static void idma64_stop_transfer(struct idma64_chan
*idma64c
)
111 struct idma64
*idma64
= to_idma64(idma64c
->vchan
.chan
.device
);
113 idma64_chan_stop(idma64
, idma64c
);
116 static void idma64_start_transfer(struct idma64_chan
*idma64c
)
118 struct idma64
*idma64
= to_idma64(idma64c
->vchan
.chan
.device
);
119 struct virt_dma_desc
*vdesc
;
121 /* Get the next descriptor */
122 vdesc
= vchan_next_desc(&idma64c
->vchan
);
124 idma64c
->desc
= NULL
;
128 list_del(&vdesc
->node
);
129 idma64c
->desc
= to_idma64_desc(vdesc
);
131 /* Configure the channel */
132 idma64_chan_init(idma64
, idma64c
);
134 /* Start the channel with a new descriptor */
135 idma64_chan_start(idma64
, idma64c
);
138 /* ---------------------------------------------------------------------- */
140 static void idma64_chan_irq(struct idma64
*idma64
, unsigned short c
,
141 u32 status_err
, u32 status_xfer
)
143 struct idma64_chan
*idma64c
= &idma64
->chan
[c
];
144 struct idma64_desc
*desc
;
146 spin_lock(&idma64c
->vchan
.lock
);
147 desc
= idma64c
->desc
;
149 if (status_err
& (1 << c
)) {
150 dma_writel(idma64
, CLEAR(ERROR
), idma64c
->mask
);
151 desc
->status
= DMA_ERROR
;
152 } else if (status_xfer
& (1 << c
)) {
153 dma_writel(idma64
, CLEAR(XFER
), idma64c
->mask
);
154 desc
->status
= DMA_COMPLETE
;
155 vchan_cookie_complete(&desc
->vdesc
);
156 idma64_start_transfer(idma64c
);
159 /* idma64_start_transfer() updates idma64c->desc */
160 if (idma64c
->desc
== NULL
|| desc
->status
== DMA_ERROR
)
161 idma64_stop_transfer(idma64c
);
163 spin_unlock(&idma64c
->vchan
.lock
);
166 static irqreturn_t
idma64_irq(int irq
, void *dev
)
168 struct idma64
*idma64
= dev
;
169 u32 status
= dma_readl(idma64
, STATUS_INT
);
174 dev_vdbg(idma64
->dma
.dev
, "%s: status=%#x\n", __func__
, status
);
176 /* Check if we have any interrupt from the DMA controller */
180 status_xfer
= dma_readl(idma64
, RAW(XFER
));
181 status_err
= dma_readl(idma64
, RAW(ERROR
));
183 for (i
= 0; i
< idma64
->dma
.chancnt
; i
++)
184 idma64_chan_irq(idma64
, i
, status_err
, status_xfer
);
189 /* ---------------------------------------------------------------------- */
191 static struct idma64_desc
*idma64_alloc_desc(unsigned int ndesc
)
193 struct idma64_desc
*desc
;
195 desc
= kzalloc(sizeof(*desc
), GFP_NOWAIT
);
199 desc
->hw
= kcalloc(ndesc
, sizeof(*desc
->hw
), GFP_NOWAIT
);
208 static void idma64_desc_free(struct idma64_chan
*idma64c
,
209 struct idma64_desc
*desc
)
211 struct idma64_hw_desc
*hw
;
214 unsigned int i
= desc
->ndesc
;
218 dma_pool_free(idma64c
->pool
, hw
->lli
, hw
->llp
);
226 static void idma64_vdesc_free(struct virt_dma_desc
*vdesc
)
228 struct idma64_chan
*idma64c
= to_idma64_chan(vdesc
->tx
.chan
);
230 idma64_desc_free(idma64c
, to_idma64_desc(vdesc
));
233 static void idma64_hw_desc_fill(struct idma64_hw_desc
*hw
,
234 struct dma_slave_config
*config
,
235 enum dma_transfer_direction direction
, u64 llp
)
237 struct idma64_lli
*lli
= hw
->lli
;
239 u32 ctlhi
= IDMA64C_CTLH_BLOCK_TS(hw
->len
);
240 u32 ctllo
= IDMA64C_CTLL_LLP_S_EN
| IDMA64C_CTLL_LLP_D_EN
;
241 u32 src_width
, dst_width
;
243 if (direction
== DMA_MEM_TO_DEV
) {
245 dar
= config
->dst_addr
;
246 ctllo
|= IDMA64C_CTLL_DST_FIX
| IDMA64C_CTLL_SRC_INC
|
248 src_width
= __ffs(sar
| hw
->len
| 4);
249 dst_width
= __ffs(config
->dst_addr_width
);
250 } else { /* DMA_DEV_TO_MEM */
251 sar
= config
->src_addr
;
253 ctllo
|= IDMA64C_CTLL_DST_INC
| IDMA64C_CTLL_SRC_FIX
|
255 src_width
= __ffs(config
->src_addr_width
);
256 dst_width
= __ffs(dar
| hw
->len
| 4);
264 IDMA64C_CTLL_SRC_MSIZE(config
->src_maxburst
) |
265 IDMA64C_CTLL_DST_MSIZE(config
->dst_maxburst
) |
266 IDMA64C_CTLL_DST_WIDTH(dst_width
) |
267 IDMA64C_CTLL_SRC_WIDTH(src_width
);
272 static void idma64_desc_fill(struct idma64_chan
*idma64c
,
273 struct idma64_desc
*desc
)
275 struct dma_slave_config
*config
= &idma64c
->config
;
276 unsigned int i
= desc
->ndesc
;
277 struct idma64_hw_desc
*hw
= &desc
->hw
[i
- 1];
278 struct idma64_lli
*lli
= hw
->lli
;
281 /* Fill the hardware descriptors and link them to a list */
284 idma64_hw_desc_fill(hw
, config
, desc
->direction
, llp
);
286 desc
->length
+= hw
->len
;
289 /* Trigger an interrupt after the last block is transfered */
290 lli
->ctllo
|= IDMA64C_CTLL_INT_EN
;
292 /* Disable LLP transfer in the last block */
293 lli
->ctllo
&= ~(IDMA64C_CTLL_LLP_S_EN
| IDMA64C_CTLL_LLP_D_EN
);
296 static struct dma_async_tx_descriptor
*idma64_prep_slave_sg(
297 struct dma_chan
*chan
, struct scatterlist
*sgl
,
298 unsigned int sg_len
, enum dma_transfer_direction direction
,
299 unsigned long flags
, void *context
)
301 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
302 struct idma64_desc
*desc
;
303 struct scatterlist
*sg
;
306 desc
= idma64_alloc_desc(sg_len
);
310 for_each_sg(sgl
, sg
, sg_len
, i
) {
311 struct idma64_hw_desc
*hw
= &desc
->hw
[i
];
313 /* Allocate DMA capable memory for hardware descriptor */
314 hw
->lli
= dma_pool_alloc(idma64c
->pool
, GFP_NOWAIT
, &hw
->llp
);
317 idma64_desc_free(idma64c
, desc
);
321 hw
->phys
= sg_dma_address(sg
);
322 hw
->len
= sg_dma_len(sg
);
325 desc
->ndesc
= sg_len
;
326 desc
->direction
= direction
;
327 desc
->status
= DMA_IN_PROGRESS
;
329 idma64_desc_fill(idma64c
, desc
);
330 return vchan_tx_prep(&idma64c
->vchan
, &desc
->vdesc
, flags
);
333 static void idma64_issue_pending(struct dma_chan
*chan
)
335 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
338 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
339 if (vchan_issue_pending(&idma64c
->vchan
) && !idma64c
->desc
)
340 idma64_start_transfer(idma64c
);
341 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
344 static size_t idma64_active_desc_size(struct idma64_chan
*idma64c
)
346 struct idma64_desc
*desc
= idma64c
->desc
;
347 struct idma64_hw_desc
*hw
;
348 size_t bytes
= desc
->length
;
349 u64 llp
= channel_readq(idma64c
, LLP
);
350 u32 ctlhi
= channel_readl(idma64c
, CTL_HI
);
358 } while (++i
< desc
->ndesc
);
363 /* The current chunk is not fully transfered yet */
364 bytes
+= desc
->hw
[--i
].len
;
366 return bytes
- IDMA64C_CTLH_BLOCK_TS(ctlhi
);
369 static enum dma_status
idma64_tx_status(struct dma_chan
*chan
,
370 dma_cookie_t cookie
, struct dma_tx_state
*state
)
372 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
373 struct virt_dma_desc
*vdesc
;
374 enum dma_status status
;
378 status
= dma_cookie_status(chan
, cookie
, state
);
379 if (status
== DMA_COMPLETE
)
382 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
383 vdesc
= vchan_find_desc(&idma64c
->vchan
, cookie
);
384 if (idma64c
->desc
&& cookie
== idma64c
->desc
->vdesc
.tx
.cookie
) {
385 bytes
= idma64_active_desc_size(idma64c
);
386 dma_set_residue(state
, bytes
);
387 status
= idma64c
->desc
->status
;
389 bytes
= to_idma64_desc(vdesc
)->length
;
390 dma_set_residue(state
, bytes
);
392 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
397 static void convert_burst(u32
*maxburst
)
400 *maxburst
= __fls(*maxburst
);
405 static int idma64_slave_config(struct dma_chan
*chan
,
406 struct dma_slave_config
*config
)
408 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
410 memcpy(&idma64c
->config
, config
, sizeof(idma64c
->config
));
412 convert_burst(&idma64c
->config
.src_maxburst
);
413 convert_burst(&idma64c
->config
.dst_maxburst
);
418 static void idma64_chan_deactivate(struct idma64_chan
*idma64c
, bool drain
)
420 unsigned short count
= 100;
423 cfglo
= channel_readl(idma64c
, CFG_LO
);
425 cfglo
|= IDMA64C_CFGL_CH_DRAIN
;
427 cfglo
&= ~IDMA64C_CFGL_CH_DRAIN
;
429 channel_writel(idma64c
, CFG_LO
, cfglo
| IDMA64C_CFGL_CH_SUSP
);
432 cfglo
= channel_readl(idma64c
, CFG_LO
);
433 } while (!(cfglo
& IDMA64C_CFGL_FIFO_EMPTY
) && --count
);
436 static void idma64_chan_activate(struct idma64_chan
*idma64c
)
440 cfglo
= channel_readl(idma64c
, CFG_LO
);
441 channel_writel(idma64c
, CFG_LO
, cfglo
& ~IDMA64C_CFGL_CH_SUSP
);
444 static int idma64_pause(struct dma_chan
*chan
)
446 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
449 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
450 if (idma64c
->desc
&& idma64c
->desc
->status
== DMA_IN_PROGRESS
) {
451 idma64_chan_deactivate(idma64c
, false);
452 idma64c
->desc
->status
= DMA_PAUSED
;
454 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
459 static int idma64_resume(struct dma_chan
*chan
)
461 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
464 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
465 if (idma64c
->desc
&& idma64c
->desc
->status
== DMA_PAUSED
) {
466 idma64c
->desc
->status
= DMA_IN_PROGRESS
;
467 idma64_chan_activate(idma64c
);
469 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
474 static int idma64_terminate_all(struct dma_chan
*chan
)
476 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
480 spin_lock_irqsave(&idma64c
->vchan
.lock
, flags
);
481 idma64_chan_deactivate(idma64c
, true);
482 idma64_stop_transfer(idma64c
);
484 idma64_vdesc_free(&idma64c
->desc
->vdesc
);
485 idma64c
->desc
= NULL
;
487 vchan_get_all_descriptors(&idma64c
->vchan
, &head
);
488 spin_unlock_irqrestore(&idma64c
->vchan
.lock
, flags
);
490 vchan_dma_desc_free_list(&idma64c
->vchan
, &head
);
494 static void idma64_synchronize(struct dma_chan
*chan
)
496 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
498 vchan_synchronize(&idma64c
->vchan
);
501 static int idma64_alloc_chan_resources(struct dma_chan
*chan
)
503 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
505 /* Create a pool of consistent memory blocks for hardware descriptors */
506 idma64c
->pool
= dma_pool_create(dev_name(chan2dev(chan
)),
508 sizeof(struct idma64_lli
), 8, 0);
509 if (!idma64c
->pool
) {
510 dev_err(chan2dev(chan
), "No memory for descriptors\n");
517 static void idma64_free_chan_resources(struct dma_chan
*chan
)
519 struct idma64_chan
*idma64c
= to_idma64_chan(chan
);
521 vchan_free_chan_resources(to_virt_chan(chan
));
522 dma_pool_destroy(idma64c
->pool
);
523 idma64c
->pool
= NULL
;
526 /* ---------------------------------------------------------------------- */
528 #define IDMA64_BUSWIDTHS \
529 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
530 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
531 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)
533 static int idma64_probe(struct idma64_chip
*chip
)
535 struct idma64
*idma64
;
536 unsigned short nr_chan
= IDMA64_NR_CHAN
;
540 idma64
= devm_kzalloc(chip
->dev
, sizeof(*idma64
), GFP_KERNEL
);
544 idma64
->regs
= chip
->regs
;
545 chip
->idma64
= idma64
;
547 idma64
->chan
= devm_kcalloc(chip
->dev
, nr_chan
, sizeof(*idma64
->chan
),
552 idma64
->all_chan_mask
= (1 << nr_chan
) - 1;
554 /* Turn off iDMA controller */
557 ret
= devm_request_irq(chip
->dev
, chip
->irq
, idma64_irq
, IRQF_SHARED
,
558 dev_name(chip
->dev
), idma64
);
562 INIT_LIST_HEAD(&idma64
->dma
.channels
);
563 for (i
= 0; i
< nr_chan
; i
++) {
564 struct idma64_chan
*idma64c
= &idma64
->chan
[i
];
566 idma64c
->vchan
.desc_free
= idma64_vdesc_free
;
567 vchan_init(&idma64c
->vchan
, &idma64
->dma
);
569 idma64c
->regs
= idma64
->regs
+ i
* IDMA64_CH_LENGTH
;
570 idma64c
->mask
= BIT(i
);
573 dma_cap_set(DMA_SLAVE
, idma64
->dma
.cap_mask
);
574 dma_cap_set(DMA_PRIVATE
, idma64
->dma
.cap_mask
);
576 idma64
->dma
.device_alloc_chan_resources
= idma64_alloc_chan_resources
;
577 idma64
->dma
.device_free_chan_resources
= idma64_free_chan_resources
;
579 idma64
->dma
.device_prep_slave_sg
= idma64_prep_slave_sg
;
581 idma64
->dma
.device_issue_pending
= idma64_issue_pending
;
582 idma64
->dma
.device_tx_status
= idma64_tx_status
;
584 idma64
->dma
.device_config
= idma64_slave_config
;
585 idma64
->dma
.device_pause
= idma64_pause
;
586 idma64
->dma
.device_resume
= idma64_resume
;
587 idma64
->dma
.device_terminate_all
= idma64_terminate_all
;
588 idma64
->dma
.device_synchronize
= idma64_synchronize
;
590 idma64
->dma
.src_addr_widths
= IDMA64_BUSWIDTHS
;
591 idma64
->dma
.dst_addr_widths
= IDMA64_BUSWIDTHS
;
592 idma64
->dma
.directions
= BIT(DMA_DEV_TO_MEM
) | BIT(DMA_MEM_TO_DEV
);
593 idma64
->dma
.residue_granularity
= DMA_RESIDUE_GRANULARITY_BURST
;
595 idma64
->dma
.dev
= chip
->dev
;
597 dma_set_max_seg_size(idma64
->dma
.dev
, IDMA64C_CTLH_BLOCK_TS_MASK
);
599 ret
= dma_async_device_register(&idma64
->dma
);
603 dev_info(chip
->dev
, "Found Intel integrated DMA 64-bit\n");
607 static int idma64_remove(struct idma64_chip
*chip
)
609 struct idma64
*idma64
= chip
->idma64
;
612 dma_async_device_unregister(&idma64
->dma
);
615 * Explicitly call devm_request_irq() to avoid the side effects with
616 * the scheduled tasklets.
618 devm_free_irq(chip
->dev
, chip
->irq
, idma64
);
620 for (i
= 0; i
< idma64
->dma
.chancnt
; i
++) {
621 struct idma64_chan
*idma64c
= &idma64
->chan
[i
];
623 tasklet_kill(&idma64c
->vchan
.task
);
629 /* ---------------------------------------------------------------------- */
631 static int idma64_platform_probe(struct platform_device
*pdev
)
633 struct idma64_chip
*chip
;
634 struct device
*dev
= &pdev
->dev
;
635 struct resource
*mem
;
638 chip
= devm_kzalloc(dev
, sizeof(*chip
), GFP_KERNEL
);
642 chip
->irq
= platform_get_irq(pdev
, 0);
646 mem
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
647 chip
->regs
= devm_ioremap_resource(dev
, mem
);
648 if (IS_ERR(chip
->regs
))
649 return PTR_ERR(chip
->regs
);
651 ret
= dma_coerce_mask_and_coherent(&pdev
->dev
, DMA_BIT_MASK(64));
657 ret
= idma64_probe(chip
);
661 platform_set_drvdata(pdev
, chip
);
665 static int idma64_platform_remove(struct platform_device
*pdev
)
667 struct idma64_chip
*chip
= platform_get_drvdata(pdev
);
669 return idma64_remove(chip
);
672 #ifdef CONFIG_PM_SLEEP
674 static int idma64_pm_suspend(struct device
*dev
)
676 struct idma64_chip
*chip
= dev_get_drvdata(dev
);
678 idma64_off(chip
->idma64
);
682 static int idma64_pm_resume(struct device
*dev
)
684 struct idma64_chip
*chip
= dev_get_drvdata(dev
);
686 idma64_on(chip
->idma64
);
690 #endif /* CONFIG_PM_SLEEP */
692 static const struct dev_pm_ops idma64_dev_pm_ops
= {
693 SET_SYSTEM_SLEEP_PM_OPS(idma64_pm_suspend
, idma64_pm_resume
)
696 static struct platform_driver idma64_platform_driver
= {
697 .probe
= idma64_platform_probe
,
698 .remove
= idma64_platform_remove
,
701 .pm
= &idma64_dev_pm_ops
,
705 module_platform_driver(idma64_platform_driver
);
707 MODULE_LICENSE("GPL v2");
708 MODULE_DESCRIPTION("iDMA64 core driver");
709 MODULE_AUTHOR("Andy Shevchenko <andriy.shevchenko@linux.intel.com>");
710 MODULE_ALIAS("platform:" DRV_NAME
);