revert-mm-fix-blkdev-size-calculation-in-generic_write_checks
[linux-2.6/linux-trees-mm.git] / drivers / dma / fsldma.c
blob71e1c32c792d916bc4ade552af7362e8f0237d8d
1 /*
2 * Freescale MPC85xx, MPC83xx DMA Engine support
4 * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved.
6 * Author:
7 * Zhang Wei <wei.zhang@freescale.com>, Jul 2007
8 * Ebony Zhu <ebony.zhu@freescale.com>, May 2007
10 * Description:
11 * DMA engine driver for Freescale MPC8540 DMA controller, which is
12 * also fit for MPC8560, MPC8555, MPC8548, MPC8641, and etc.
13 * The support for MPC8349 DMA contorller is also added.
15 * This is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
22 #include <linux/init.h>
23 #include <linux/module.h>
24 #include <linux/pci.h>
25 #include <linux/interrupt.h>
26 #include <linux/dmaengine.h>
27 #include <linux/delay.h>
28 #include <linux/dma-mapping.h>
29 #include <linux/dmapool.h>
30 #include <linux/of_platform.h>
32 #include "fsldma.h"
34 static void dma_init(struct fsl_dma_chan *fsl_chan)
36 /* Reset the channel */
37 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 0, 32);
39 switch (fsl_chan->feature & FSL_DMA_IP_MASK) {
40 case FSL_DMA_IP_85XX:
41 /* Set the channel to below modes:
42 * EIE - Error interrupt enable
43 * EOSIE - End of segments interrupt enable (basic mode)
44 * EOLNIE - End of links interrupt enable
46 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EIE
47 | FSL_DMA_MR_EOLNIE | FSL_DMA_MR_EOSIE, 32);
48 break;
49 case FSL_DMA_IP_83XX:
50 /* Set the channel to below modes:
51 * EOTIE - End-of-transfer interrupt enable
53 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE,
54 32);
55 break;
60 static void set_sr(struct fsl_dma_chan *fsl_chan, dma_addr_t val)
62 DMA_OUT(fsl_chan, &fsl_chan->reg_base->sr, val, 32);
65 static dma_addr_t get_sr(struct fsl_dma_chan *fsl_chan)
67 return DMA_IN(fsl_chan, &fsl_chan->reg_base->sr, 32);
70 static void set_desc_cnt(struct fsl_dma_chan *fsl_chan,
71 struct fsl_dma_ld_hw *hw, u32 count)
73 hw->count = CPU_TO_DMA(fsl_chan, count, 32);
76 static void set_desc_src(struct fsl_dma_chan *fsl_chan,
77 struct fsl_dma_ld_hw *hw, dma_addr_t src)
79 u64 snoop_bits;
81 snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
82 ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0;
83 hw->src_addr = CPU_TO_DMA(fsl_chan, snoop_bits | src, 64);
86 static void set_desc_dest(struct fsl_dma_chan *fsl_chan,
87 struct fsl_dma_ld_hw *hw, dma_addr_t dest)
89 u64 snoop_bits;
91 snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX)
92 ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0;
93 hw->dst_addr = CPU_TO_DMA(fsl_chan, snoop_bits | dest, 64);
96 static void set_desc_next(struct fsl_dma_chan *fsl_chan,
97 struct fsl_dma_ld_hw *hw, dma_addr_t next)
99 u64 snoop_bits;
101 snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX)
102 ? FSL_DMA_SNEN : 0;
103 hw->next_ln_addr = CPU_TO_DMA(fsl_chan, snoop_bits | next, 64);
106 static void set_cdar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr)
108 DMA_OUT(fsl_chan, &fsl_chan->reg_base->cdar, addr | FSL_DMA_SNEN, 64);
111 static dma_addr_t get_cdar(struct fsl_dma_chan *fsl_chan)
113 return DMA_IN(fsl_chan, &fsl_chan->reg_base->cdar, 64) & ~FSL_DMA_SNEN;
116 static void set_ndar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr)
118 DMA_OUT(fsl_chan, &fsl_chan->reg_base->ndar, addr, 64);
121 static dma_addr_t get_ndar(struct fsl_dma_chan *fsl_chan)
123 return DMA_IN(fsl_chan, &fsl_chan->reg_base->ndar, 64);
126 static int dma_is_idle(struct fsl_dma_chan *fsl_chan)
128 u32 sr = get_sr(fsl_chan);
129 return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH);
132 static void dma_start(struct fsl_dma_chan *fsl_chan)
134 u32 mr_set = 0;;
136 if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) {
137 DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32);
138 mr_set |= FSL_DMA_MR_EMP_EN;
139 } else
140 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
141 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
142 & ~FSL_DMA_MR_EMP_EN, 32);
144 if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT)
145 mr_set |= FSL_DMA_MR_EMS_EN;
146 else
147 mr_set |= FSL_DMA_MR_CS;
149 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
150 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
151 | mr_set, 32);
154 static void dma_halt(struct fsl_dma_chan *fsl_chan)
156 int i = 0;
157 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
158 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | FSL_DMA_MR_CA,
159 32);
160 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
161 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & ~(FSL_DMA_MR_CS
162 | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA), 32);
164 while (!dma_is_idle(fsl_chan) && (i++ < 100))
165 udelay(10);
166 if (i >= 100 && !dma_is_idle(fsl_chan))
167 dev_err(fsl_chan->dev, "DMA halt timeout!\n");
170 static void set_ld_eol(struct fsl_dma_chan *fsl_chan,
171 struct fsl_desc_sw *desc)
173 desc->hw.next_ln_addr = CPU_TO_DMA(fsl_chan,
174 DMA_TO_CPU(fsl_chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL,
175 64);
178 static void append_ld_queue(struct fsl_dma_chan *fsl_chan,
179 struct fsl_desc_sw *new_desc)
181 struct fsl_desc_sw *queue_tail = to_fsl_desc(fsl_chan->ld_queue.prev);
183 if (list_empty(&fsl_chan->ld_queue))
184 return;
186 /* Link to the new descriptor physical address and
187 * Enable End-of-segment interrupt for
188 * the last link descriptor.
189 * (the previous node's next link descriptor)
191 * For FSL_DMA_IP_83xx, the snoop enable bit need be set.
193 queue_tail->hw.next_ln_addr = CPU_TO_DMA(fsl_chan,
194 new_desc->async_tx.phys | FSL_DMA_EOSIE |
195 (((fsl_chan->feature & FSL_DMA_IP_MASK)
196 == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0), 64);
200 * fsl_chan_set_src_loop_size - Set source address hold transfer size
201 * @fsl_chan : Freescale DMA channel
202 * @size : Address loop size, 0 for disable loop
204 * The set source address hold transfer size. The source
205 * address hold or loop transfer size is when the DMA transfer
206 * data from source address (SA), if the loop size is 4, the DMA will
207 * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA,
208 * SA + 1 ... and so on.
210 static void fsl_chan_set_src_loop_size(struct fsl_dma_chan *fsl_chan, int size)
212 switch (size) {
213 case 0:
214 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
215 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) &
216 (~FSL_DMA_MR_SAHE), 32);
217 break;
218 case 1:
219 case 2:
220 case 4:
221 case 8:
222 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
223 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) |
224 FSL_DMA_MR_SAHE | (__ilog2(size) << 14),
225 32);
226 break;
231 * fsl_chan_set_dest_loop_size - Set destination address hold transfer size
232 * @fsl_chan : Freescale DMA channel
233 * @size : Address loop size, 0 for disable loop
235 * The set destination address hold transfer size. The destination
236 * address hold or loop transfer size is when the DMA transfer
237 * data to destination address (TA), if the loop size is 4, the DMA will
238 * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA,
239 * TA + 1 ... and so on.
241 static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size)
243 switch (size) {
244 case 0:
245 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
246 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) &
247 (~FSL_DMA_MR_DAHE), 32);
248 break;
249 case 1:
250 case 2:
251 case 4:
252 case 8:
253 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
254 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) |
255 FSL_DMA_MR_DAHE | (__ilog2(size) << 16),
256 32);
257 break;
262 * fsl_chan_toggle_ext_pause - Toggle channel external pause status
263 * @fsl_chan : Freescale DMA channel
264 * @size : Pause control size, 0 for disable external pause control.
265 * The maximum is 1024.
267 * The Freescale DMA channel can be controlled by the external
268 * signal DREQ#. The pause control size is how many bytes are allowed
269 * to transfer before pausing the channel, after which a new assertion
270 * of DREQ# resumes channel operation.
272 static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int size)
274 if (size > 1024)
275 return;
277 if (size) {
278 DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
279 DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
280 | ((__ilog2(size) << 24) & 0x0f000000),
281 32);
282 fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
283 } else
284 fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
288 * fsl_chan_toggle_ext_start - Toggle channel external start status
289 * @fsl_chan : Freescale DMA channel
290 * @enable : 0 is disabled, 1 is enabled.
292 * If enable the external start, the channel can be started by an
293 * external DMA start pin. So the dma_start() does not start the
294 * transfer immediately. The DMA channel will wait for the
295 * control pin asserted.
297 static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable)
299 if (enable)
300 fsl_chan->feature |= FSL_DMA_CHAN_START_EXT;
301 else
302 fsl_chan->feature &= ~FSL_DMA_CHAN_START_EXT;
305 static void fsl_dma_set_src(dma_addr_t addr,
306 struct dma_async_tx_descriptor *tx, int index)
308 struct fsl_desc_sw *desc_node, *desc = tx_to_fsl_desc(tx);
309 struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
311 list_for_each_entry(desc_node, &desc->async_tx.tx_list, node) {
312 set_desc_src(fsl_chan, &desc_node->hw, addr);
313 addr += FSL_DMA_BCR_MAX_CNT;
317 static void fsl_dma_set_dest(dma_addr_t addr,
318 struct dma_async_tx_descriptor *tx, int index)
320 struct fsl_desc_sw *desc_node, *desc = tx_to_fsl_desc(tx);
321 struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
323 list_for_each_entry(desc_node, &desc->async_tx.tx_list, node) {
324 set_desc_dest(fsl_chan, &desc_node->hw, addr);
325 addr += FSL_DMA_BCR_MAX_CNT;
329 static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
331 struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
332 struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
333 unsigned long flags;
334 dma_cookie_t cookie;
336 /* cookie increment and adding to ld_queue must be atomic */
337 spin_lock_irqsave(&fsl_chan->desc_lock, flags);
339 cookie = fsl_chan->common.cookie;
340 cookie++;
341 if (cookie < 0)
342 cookie = 1;
343 desc->async_tx.cookie = cookie;
344 fsl_chan->common.cookie = desc->async_tx.cookie;
346 append_ld_queue(fsl_chan, desc);
347 list_splice_init(&desc->async_tx.tx_list, fsl_chan->ld_queue.prev);
349 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
351 return cookie;
355 * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool.
356 * @fsl_chan : Freescale DMA channel
358 * Return - The descriptor allocated. NULL for failed.
360 static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
361 struct fsl_dma_chan *fsl_chan)
363 dma_addr_t pdesc;
364 struct fsl_desc_sw *desc_sw;
366 desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc);
367 if (desc_sw) {
368 memset(desc_sw, 0, sizeof(struct fsl_desc_sw));
369 dma_async_tx_descriptor_init(&desc_sw->async_tx,
370 &fsl_chan->common);
371 desc_sw->async_tx.tx_set_src = fsl_dma_set_src;
372 desc_sw->async_tx.tx_set_dest = fsl_dma_set_dest;
373 desc_sw->async_tx.tx_submit = fsl_dma_tx_submit;
374 INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
375 desc_sw->async_tx.phys = pdesc;
378 return desc_sw;
383 * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel.
384 * @fsl_chan : Freescale DMA channel
386 * This function will create a dma pool for descriptor allocation.
388 * Return - The number of descriptors allocated.
390 static int fsl_dma_alloc_chan_resources(struct dma_chan *chan)
392 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan);
393 LIST_HEAD(tmp_list);
395 /* We need the descriptor to be aligned to 32bytes
396 * for meeting FSL DMA specification requirement.
398 fsl_chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool",
399 fsl_chan->dev, sizeof(struct fsl_desc_sw),
400 32, 0);
401 if (!fsl_chan->desc_pool) {
402 dev_err(fsl_chan->dev, "No memory for channel %d "
403 "descriptor dma pool.\n", fsl_chan->id);
404 return 0;
407 return 1;
411 * fsl_dma_free_chan_resources - Free all resources of the channel.
412 * @fsl_chan : Freescale DMA channel
414 static void fsl_dma_free_chan_resources(struct dma_chan *chan)
416 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan);
417 struct fsl_desc_sw *desc, *_desc;
418 unsigned long flags;
420 dev_dbg(fsl_chan->dev, "Free all channel resources.\n");
421 spin_lock_irqsave(&fsl_chan->desc_lock, flags);
422 list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) {
423 #ifdef FSL_DMA_LD_DEBUG
424 dev_dbg(fsl_chan->dev,
425 "LD %p will be released.\n", desc);
426 #endif
427 list_del(&desc->node);
428 /* free link descriptor */
429 dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
431 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
432 dma_pool_destroy(fsl_chan->desc_pool);
435 static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
436 struct dma_chan *chan, size_t len, int int_en)
438 struct fsl_dma_chan *fsl_chan;
439 struct fsl_desc_sw *first = NULL, *prev = NULL, *new;
440 size_t copy;
441 LIST_HEAD(link_chain);
443 if (!chan)
444 return NULL;
446 if (!len)
447 return NULL;
449 fsl_chan = to_fsl_chan(chan);
451 do {
453 /* Allocate the link descriptor from DMA pool */
454 new = fsl_dma_alloc_descriptor(fsl_chan);
455 if (!new) {
456 dev_err(fsl_chan->dev,
457 "No free memory for link descriptor\n");
458 return NULL;
460 #ifdef FSL_DMA_LD_DEBUG
461 dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
462 #endif
464 copy = min(len, FSL_DMA_BCR_MAX_CNT);
466 set_desc_cnt(fsl_chan, &new->hw, copy);
468 if (!first)
469 first = new;
470 else
471 set_desc_next(fsl_chan, &prev->hw, new->async_tx.phys);
473 new->async_tx.cookie = 0;
474 new->async_tx.ack = 1;
476 prev = new;
477 len -= copy;
479 /* Insert the link descriptor to the LD ring */
480 list_add_tail(&new->node, &first->async_tx.tx_list);
481 } while (len);
483 new->async_tx.ack = 0; /* client is in control of this ack */
484 new->async_tx.cookie = -EBUSY;
486 /* Set End-of-link to the last link descriptor of new list*/
487 set_ld_eol(fsl_chan, new);
489 return first ? &first->async_tx : NULL;
493 * fsl_dma_update_completed_cookie - Update the completed cookie.
494 * @fsl_chan : Freescale DMA channel
496 static void fsl_dma_update_completed_cookie(struct fsl_dma_chan *fsl_chan)
498 struct fsl_desc_sw *cur_desc;
499 dma_addr_t ld_phy;
501 ld_phy = get_cdar(fsl_chan) & FSL_DMA_NLDA_MASK;
503 if (ld_phy) {
504 cur_desc = (struct fsl_desc_sw *)bus_to_virt(ld_phy);
506 if (cur_desc->async_tx.cookie) {
507 if (dma_is_idle(fsl_chan))
508 fsl_chan->completed_cookie =
509 cur_desc->async_tx.cookie;
510 else
511 fsl_chan->completed_cookie =
512 cur_desc->async_tx.cookie - 1;
518 * fsl_chan_ld_cleanup - Clean up link descriptors
519 * @fsl_chan : Freescale DMA channel
521 * This function clean up the ld_queue of DMA channel.
522 * If 'in_intr' is set, the function will move the link descriptor to
523 * the recycle list. Otherwise, free it directly.
525 static void fsl_chan_ld_cleanup(struct fsl_dma_chan *fsl_chan)
527 struct fsl_desc_sw *desc, *_desc;
528 unsigned long flags;
530 spin_lock_irqsave(&fsl_chan->desc_lock, flags);
532 fsl_dma_update_completed_cookie(fsl_chan);
533 dev_dbg(fsl_chan->dev, "chan completed_cookie = %d\n",
534 fsl_chan->completed_cookie);
535 list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) {
536 dma_async_tx_callback callback;
537 void *callback_param;
539 if (dma_async_is_complete(desc->async_tx.cookie,
540 fsl_chan->completed_cookie, fsl_chan->common.cookie)
541 == DMA_IN_PROGRESS)
542 break;
544 callback = desc->async_tx.callback;
545 callback_param = desc->async_tx.callback_param;
547 /* Remove from ld_queue list */
548 list_del(&desc->node);
550 dev_dbg(fsl_chan->dev, "link descriptor %p will be recycle.\n",
551 desc);
552 dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
554 /* Run the link descriptor callback function */
555 if (callback) {
556 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
557 dev_dbg(fsl_chan->dev, "link descriptor %p callback\n",
558 desc);
559 callback(callback_param);
560 spin_lock_irqsave(&fsl_chan->desc_lock, flags);
563 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
567 * fsl_chan_xfer_ld_queue - Transfer link descriptors in channel ld_queue.
568 * @fsl_chan : Freescale DMA channel
570 static void fsl_chan_xfer_ld_queue(struct fsl_dma_chan *fsl_chan)
572 struct list_head *ld_node;
573 dma_addr_t next_dest_addr;
574 unsigned long flags;
576 if (!dma_is_idle(fsl_chan))
577 return;
579 dma_halt(fsl_chan);
581 /* If there are some link descriptors
582 * not transfered in queue. We need to start it.
584 spin_lock_irqsave(&fsl_chan->desc_lock, flags);
586 /* Find the first un-transfer desciptor */
587 for (ld_node = fsl_chan->ld_queue.next;
588 (ld_node != &fsl_chan->ld_queue)
589 && (dma_async_is_complete(
590 to_fsl_desc(ld_node)->async_tx.cookie,
591 fsl_chan->completed_cookie,
592 fsl_chan->common.cookie) == DMA_SUCCESS);
593 ld_node = ld_node->next);
595 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
597 if (ld_node != &fsl_chan->ld_queue) {
598 /* Get the ld start address from ld_queue */
599 next_dest_addr = to_fsl_desc(ld_node)->async_tx.phys;
600 dev_dbg(fsl_chan->dev, "xfer LDs staring from 0x%016llx\n",
601 (u64)next_dest_addr);
602 set_cdar(fsl_chan, next_dest_addr);
603 dma_start(fsl_chan);
604 } else {
605 set_cdar(fsl_chan, 0);
606 set_ndar(fsl_chan, 0);
611 * fsl_dma_memcpy_issue_pending - Issue the DMA start command
612 * @fsl_chan : Freescale DMA channel
614 static void fsl_dma_memcpy_issue_pending(struct dma_chan *chan)
616 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan);
618 #ifdef FSL_DMA_LD_DEBUG
619 struct fsl_desc_sw *ld;
620 unsigned long flags;
622 spin_lock_irqsave(&fsl_chan->desc_lock, flags);
623 if (list_empty(&fsl_chan->ld_queue)) {
624 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
625 return;
628 dev_dbg(fsl_chan->dev, "--memcpy issue--\n");
629 list_for_each_entry(ld, &fsl_chan->ld_queue, node) {
630 int i;
631 dev_dbg(fsl_chan->dev, "Ch %d, LD %08x\n",
632 fsl_chan->id, ld->async_tx.phys);
633 for (i = 0; i < 8; i++)
634 dev_dbg(fsl_chan->dev, "LD offset %d: %08x\n",
635 i, *(((u32 *)&ld->hw) + i));
637 dev_dbg(fsl_chan->dev, "----------------\n");
638 spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
639 #endif
641 fsl_chan_xfer_ld_queue(fsl_chan);
644 static void fsl_dma_dependency_added(struct dma_chan *chan)
646 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan);
648 fsl_chan_ld_cleanup(fsl_chan);
652 * fsl_dma_is_complete - Determine the DMA status
653 * @fsl_chan : Freescale DMA channel
655 static enum dma_status fsl_dma_is_complete(struct dma_chan *chan,
656 dma_cookie_t cookie,
657 dma_cookie_t *done,
658 dma_cookie_t *used)
660 struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan);
661 dma_cookie_t last_used;
662 dma_cookie_t last_complete;
664 fsl_chan_ld_cleanup(fsl_chan);
666 last_used = chan->cookie;
667 last_complete = fsl_chan->completed_cookie;
669 if (done)
670 *done = last_complete;
672 if (used)
673 *used = last_used;
675 return dma_async_is_complete(cookie, last_complete, last_used);
678 static irqreturn_t fsl_dma_chan_do_interrupt(int irq, void *data)
680 struct fsl_dma_chan *fsl_chan = (struct fsl_dma_chan *)data;
681 dma_addr_t stat;
683 stat = get_sr(fsl_chan);
684 dev_dbg(fsl_chan->dev, "event: channel %d, stat = 0x%x\n",
685 fsl_chan->id, stat);
686 set_sr(fsl_chan, stat); /* Clear the event register */
688 stat &= ~(FSL_DMA_SR_CB | FSL_DMA_SR_CH);
689 if (!stat)
690 return IRQ_NONE;
692 if (stat & FSL_DMA_SR_TE)
693 dev_err(fsl_chan->dev, "Transfer Error!\n");
695 /* If the link descriptor segment transfer finishes,
696 * we will recycle the used descriptor.
698 if (stat & FSL_DMA_SR_EOSI) {
699 dev_dbg(fsl_chan->dev, "event: End-of-segments INT\n");
700 dev_dbg(fsl_chan->dev, "event: clndar 0x%016llx, "
701 "nlndar 0x%016llx\n", (u64)get_cdar(fsl_chan),
702 (u64)get_ndar(fsl_chan));
703 stat &= ~FSL_DMA_SR_EOSI;
704 fsl_chan_ld_cleanup(fsl_chan);
707 /* If it current transfer is the end-of-transfer,
708 * we should clear the Channel Start bit for
709 * prepare next transfer.
711 if (stat & (FSL_DMA_SR_EOLNI | FSL_DMA_SR_EOCDI)) {
712 dev_dbg(fsl_chan->dev, "event: End-of-link INT\n");
713 stat &= ~FSL_DMA_SR_EOLNI;
714 fsl_chan_xfer_ld_queue(fsl_chan);
717 if (stat)
718 dev_dbg(fsl_chan->dev, "event: unhandled sr 0x%02x\n",
719 stat);
721 dev_dbg(fsl_chan->dev, "event: Exit\n");
722 tasklet_schedule(&fsl_chan->tasklet);
723 return IRQ_HANDLED;
726 static irqreturn_t fsl_dma_do_interrupt(int irq, void *data)
728 struct fsl_dma_device *fdev = (struct fsl_dma_device *)data;
729 u32 gsr;
730 int ch_nr;
732 gsr = (fdev->feature & FSL_DMA_BIG_ENDIAN) ? in_be32(fdev->reg_base)
733 : in_le32(fdev->reg_base);
734 ch_nr = (32 - ffs(gsr)) / 8;
736 return fdev->chan[ch_nr] ? fsl_dma_chan_do_interrupt(irq,
737 fdev->chan[ch_nr]) : IRQ_NONE;
740 static void dma_do_tasklet(unsigned long data)
742 struct fsl_dma_chan *fsl_chan = (struct fsl_dma_chan *)data;
743 fsl_chan_ld_cleanup(fsl_chan);
746 static void fsl_dma_callback_test(struct fsl_dma_chan *fsl_chan)
748 if (fsl_chan)
749 dev_info(fsl_chan->dev, "selftest: callback is ok!\n");
752 static int fsl_dma_self_test(struct fsl_dma_chan *fsl_chan)
754 struct dma_chan *chan;
755 int err = 0;
756 dma_addr_t addr;
757 dma_cookie_t cookie;
758 u8 *src, *dest;
759 int i;
760 size_t test_size;
761 struct dma_async_tx_descriptor *tx1, *tx2, *tx3;
763 test_size = 4096;
765 src = kmalloc(test_size * 2, GFP_KERNEL);
766 if (!src) {
767 dev_err(fsl_chan->dev,
768 "selftest: Cannot alloc memory for test!\n");
769 err = -ENOMEM;
770 goto out;
773 dest = src + test_size;
775 for (i = 0; i < test_size; i++)
776 src[i] = (u8) i;
778 chan = &fsl_chan->common;
780 if (fsl_dma_alloc_chan_resources(chan) < 1) {
781 dev_err(fsl_chan->dev,
782 "selftest: Cannot alloc resources for DMA\n");
783 err = -ENODEV;
784 goto out;
787 /* TX 1 */
788 tx1 = fsl_dma_prep_memcpy(chan, test_size / 2, 0);
789 async_tx_ack(tx1);
790 addr = dma_map_single(fsl_chan->dev, src, test_size / 2, DMA_TO_DEVICE);
791 fsl_dma_set_src(addr, tx1, 0);
792 addr = dma_map_single(fsl_chan->dev, dest, test_size / 2,
793 DMA_FROM_DEVICE);
794 fsl_dma_set_dest(addr, tx1, 0);
796 cookie = fsl_dma_tx_submit(tx1);
797 fsl_dma_memcpy_issue_pending(chan);
798 msleep(2);
800 if (fsl_dma_is_complete(chan, cookie, NULL, NULL) != DMA_SUCCESS) {
801 dev_err(fsl_chan->dev, "selftest: Time out!\n");
802 err = -ENODEV;
803 goto out;
806 /* Test free and re-alloc channel resources */
807 fsl_dma_free_chan_resources(chan);
809 if (fsl_dma_alloc_chan_resources(chan) < 1) {
810 dev_err(fsl_chan->dev,
811 "selftest: Cannot alloc resources for DMA\n");
812 err = -ENODEV;
813 goto free_resources;
816 /* Continue to test
817 * TX 2
819 tx2 = fsl_dma_prep_memcpy(chan, test_size / 4, 0);
820 async_tx_ack(tx2);
821 addr = dma_map_single(fsl_chan->dev, src + test_size / 2,
822 test_size / 4, DMA_TO_DEVICE);
823 fsl_dma_set_src(addr, tx2, 0);
824 addr = dma_map_single(fsl_chan->dev, dest + test_size / 2,
825 test_size / 4, DMA_FROM_DEVICE);
826 fsl_dma_set_dest(addr, tx2, 0);
828 /* TX 3 */
829 tx3 = fsl_dma_prep_memcpy(chan, test_size / 4, 0);
830 async_tx_ack(tx3);
831 addr = dma_map_single(fsl_chan->dev, src + test_size * 3 / 4,
832 test_size / 4, DMA_TO_DEVICE);
833 fsl_dma_set_src(addr, tx3, 0);
834 addr = dma_map_single(fsl_chan->dev, dest + test_size * 3 / 4,
835 test_size / 4, DMA_FROM_DEVICE);
836 fsl_dma_set_dest(addr, tx3, 0);
838 /* Test exchanging the prepared tx sort */
839 cookie = fsl_dma_tx_submit(tx3);
840 cookie = fsl_dma_tx_submit(tx2);
842 #ifdef FSL_DMA_CALLBACKTEST
843 if (dma_has_cap(DMA_INTERRUPT, ((struct fsl_dma_device *)
844 dev_get_drvdata(fsl_chan->dev->parent))->common.cap_mask)) {
845 tx3->callback = fsl_dma_callback_test;
846 tx3->callback_param = fsl_chan;
848 #endif
849 fsl_dma_memcpy_issue_pending(chan);
850 msleep(2);
852 if (fsl_dma_is_complete(chan, cookie, NULL, NULL) != DMA_SUCCESS) {
853 dev_err(fsl_chan->dev, "selftest: Time out!\n");
854 err = -ENODEV;
855 goto free_resources;
858 err = memcmp(src, dest, test_size);
859 if (err) {
860 for (i = 0; (*(src + i) == *(dest + i)) && (i < test_size);
861 i++);
862 dev_err(fsl_chan->dev, "selftest: Test failed, data %d/%d is "
863 "error! src 0x%x, dest 0x%x\n",
864 i, test_size, *(src + i), *(dest + i));
867 free_resources:
868 fsl_dma_free_chan_resources(chan);
869 out:
870 kfree(src);
871 return err;
874 static int __devinit of_fsl_dma_chan_probe(struct of_device *dev,
875 const struct of_device_id *match)
877 struct fsl_dma_device *fdev;
878 struct fsl_dma_chan *new_fsl_chan;
879 int err;
881 fdev = dev_get_drvdata(dev->dev.parent);
882 BUG_ON(!fdev);
884 /* alloc channel */
885 new_fsl_chan = kzalloc(sizeof(struct fsl_dma_chan), GFP_KERNEL);
886 if (!new_fsl_chan) {
887 dev_err(&dev->dev, "No free memory for allocating "
888 "dma channels!\n");
889 err = -ENOMEM;
890 goto err;
893 /* get dma channel register base */
894 err = of_address_to_resource(dev->node, 0, &new_fsl_chan->reg);
895 if (err) {
896 dev_err(&dev->dev, "Can't get %s property 'reg'\n",
897 dev->node->full_name);
898 goto err;
901 new_fsl_chan->feature = *(u32 *)match->data;
903 if (!fdev->feature)
904 fdev->feature = new_fsl_chan->feature;
906 /* If the DMA device's feature is different than its channels',
907 * report the bug.
909 WARN_ON(fdev->feature != new_fsl_chan->feature);
911 new_fsl_chan->dev = &dev->dev;
912 new_fsl_chan->reg_base = ioremap(new_fsl_chan->reg.start,
913 new_fsl_chan->reg.end - new_fsl_chan->reg.start + 1);
915 new_fsl_chan->id = ((new_fsl_chan->reg.start - 0x100) & 0xfff) >> 7;
916 if (new_fsl_chan->id > FSL_DMA_MAX_CHANS_PER_DEVICE) {
917 dev_err(&dev->dev, "There is no %d channel!\n",
918 new_fsl_chan->id);
919 err = -EINVAL;
920 goto err;
922 fdev->chan[new_fsl_chan->id] = new_fsl_chan;
923 tasklet_init(&new_fsl_chan->tasklet, dma_do_tasklet,
924 (unsigned long)new_fsl_chan);
926 /* Init the channel */
927 dma_init(new_fsl_chan);
929 /* Clear cdar registers */
930 set_cdar(new_fsl_chan, 0);
932 switch (new_fsl_chan->feature & FSL_DMA_IP_MASK) {
933 case FSL_DMA_IP_85XX:
934 new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
935 new_fsl_chan->toggle_ext_pause = fsl_chan_toggle_ext_pause;
936 case FSL_DMA_IP_83XX:
937 new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size;
938 new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size;
941 spin_lock_init(&new_fsl_chan->desc_lock);
942 INIT_LIST_HEAD(&new_fsl_chan->ld_queue);
944 new_fsl_chan->common.device = &fdev->common;
946 /* Add the channel to DMA device channel list */
947 list_add_tail(&new_fsl_chan->common.device_node,
948 &fdev->common.channels);
949 fdev->common.chancnt++;
951 new_fsl_chan->irq = irq_of_parse_and_map(dev->node, 0);
952 if (new_fsl_chan->irq != NO_IRQ) {
953 err = request_irq(new_fsl_chan->irq,
954 &fsl_dma_chan_do_interrupt, IRQF_SHARED,
955 "fsldma-channel", new_fsl_chan);
956 if (err) {
957 dev_err(&dev->dev, "DMA channel %s request_irq error "
958 "with return %d\n", dev->node->full_name, err);
959 goto err;
963 #ifdef CONFIG_FSL_DMA_SELFTEST
964 err = fsl_dma_self_test(new_fsl_chan);
965 if (err)
966 goto err;
967 #endif
969 dev_info(&dev->dev, "#%d (%s), irq %d\n", new_fsl_chan->id,
970 match->compatible, new_fsl_chan->irq);
972 return 0;
973 err:
974 dma_halt(new_fsl_chan);
975 iounmap(new_fsl_chan->reg_base);
976 free_irq(new_fsl_chan->irq, new_fsl_chan);
977 list_del(&new_fsl_chan->common.device_node);
978 kfree(new_fsl_chan);
979 return err;
982 const u32 mpc8540_dma_ip_feature = FSL_DMA_IP_85XX | FSL_DMA_BIG_ENDIAN;
983 const u32 mpc8349_dma_ip_feature = FSL_DMA_IP_83XX | FSL_DMA_LITTLE_ENDIAN;
985 static struct of_device_id of_fsl_dma_chan_ids[] = {
987 .compatible = "fsl,mpc8540-dma-channel",
988 .data = (void *)&mpc8540_dma_ip_feature,
991 .compatible = "fsl,mpc8349-dma-channel",
992 .data = (void *)&mpc8349_dma_ip_feature,
997 static struct of_platform_driver of_fsl_dma_chan_driver = {
998 .name = "of-fsl-dma-channel",
999 .match_table = of_fsl_dma_chan_ids,
1000 .probe = of_fsl_dma_chan_probe,
1003 static __init int of_fsl_dma_chan_init(void)
1005 return of_register_platform_driver(&of_fsl_dma_chan_driver);
1008 static int __devinit of_fsl_dma_probe(struct of_device *dev,
1009 const struct of_device_id *match)
1011 int err;
1012 unsigned int irq;
1013 struct fsl_dma_device *fdev;
1015 fdev = kzalloc(sizeof(struct fsl_dma_device), GFP_KERNEL);
1016 if (!fdev) {
1017 dev_err(&dev->dev, "No enough memory for 'priv'\n");
1018 err = -ENOMEM;
1019 goto err;
1021 fdev->dev = &dev->dev;
1022 INIT_LIST_HEAD(&fdev->common.channels);
1024 /* get DMA controller register base */
1025 err = of_address_to_resource(dev->node, 0, &fdev->reg);
1026 if (err) {
1027 dev_err(&dev->dev, "Can't get %s property 'reg'\n",
1028 dev->node->full_name);
1029 goto err;
1032 dev_info(&dev->dev, "Probe the Freescale DMA driver for %s "
1033 "controller at 0x%08x...\n",
1034 match->compatible, fdev->reg.start);
1035 fdev->reg_base = ioremap(fdev->reg.start, fdev->reg.end
1036 - fdev->reg.start + 1);
1038 dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
1039 dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
1040 fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
1041 fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
1042 fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
1043 fdev->common.device_is_tx_complete = fsl_dma_is_complete;
1044 fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
1045 fdev->common.device_dependency_added = fsl_dma_dependency_added;
1046 fdev->common.dev = &dev->dev;
1048 irq = irq_of_parse_and_map(dev->node, 0);
1049 if (irq != NO_IRQ) {
1050 err = request_irq(irq, &fsl_dma_do_interrupt, IRQF_SHARED,
1051 "fsldma-device", fdev);
1052 if (err) {
1053 dev_err(&dev->dev, "DMA device request_irq error "
1054 "with return %d\n", err);
1055 goto err;
1059 dev_set_drvdata(&(dev->dev), fdev);
1060 of_platform_bus_probe(dev->node, of_fsl_dma_chan_ids, &dev->dev);
1062 dma_async_device_register(&fdev->common);
1063 return 0;
1065 err:
1066 iounmap(fdev->reg_base);
1067 kfree(fdev);
1068 return err;
1071 static struct of_device_id of_fsl_dma_ids[] = {
1072 { .compatible = "fsl,mpc8540-dma", },
1073 { .compatible = "fsl,mpc8349-dma", },
1077 static struct of_platform_driver of_fsl_dma_driver = {
1078 .name = "of-fsl-dma",
1079 .match_table = of_fsl_dma_ids,
1080 .probe = of_fsl_dma_probe,
1083 static __init int of_fsl_dma_init(void)
1085 return of_register_platform_driver(&of_fsl_dma_driver);
1088 subsys_initcall(of_fsl_dma_chan_init);
1089 subsys_initcall(of_fsl_dma_init);