2 * Copyright (C) 2005-2006 by Texas Instruments
4 * This file implements a DMA interface using TI's CPPI DMA.
5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
6 * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
9 #include <linux/module.h>
10 #include <linux/platform_device.h>
11 #include <linux/slab.h>
12 #include <linux/usb.h>
14 #include "musb_core.h"
15 #include "musb_debug.h"
19 /* CPPI DMA status 7-mar-2006:
21 * - See musb_{host,gadget}.c for more info
23 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
24 * which can easily saturate the CPU under non-mass-storage loads.
26 * NOTES 24-aug-2006 (2.6.18-rc4):
28 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
29 * evidently after the 1 byte packet was received and acked, the queue
30 * of BDs got garbaged so it wouldn't empty the fifo. (rxcsr 0x2003,
31 * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
32 * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
33 * of its next (512 byte) packet. IRQ issues?
35 * REVISIT: the "transfer DMA" glue between CPPI and USB fifos will
36 * evidently also directly update the RX and TX CSRs ... so audit all
37 * host and peripheral side DMA code to avoid CSR access after DMA has
41 /* REVISIT now we can avoid preallocating these descriptors; or
42 * more simply, switch to a global freelist not per-channel ones.
43 * Note: at full speed, 64 descriptors == 4K bulk data.
45 #define NUM_TXCHAN_BD 64
46 #define NUM_RXCHAN_BD 64
48 static inline void cpu_drain_writebuffer(void)
51 #ifdef CONFIG_CPU_ARM926T
52 /* REVISIT this "should not be needed",
53 * but lack of it sure seemed to hurt ...
55 asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
59 static inline struct cppi_descriptor
*cppi_bd_alloc(struct cppi_channel
*c
)
61 struct cppi_descriptor
*bd
= c
->freelist
;
64 c
->freelist
= bd
->next
;
69 cppi_bd_free(struct cppi_channel
*c
, struct cppi_descriptor
*bd
)
73 bd
->next
= c
->freelist
;
78 * Start DMA controller
80 * Initialize the DMA controller as necessary.
83 /* zero out entire rx state RAM entry for the channel */
84 static void cppi_reset_rx(struct cppi_rx_stateram __iomem
*rx
)
86 musb_writel(&rx
->rx_skipbytes
, 0, 0);
87 musb_writel(&rx
->rx_head
, 0, 0);
88 musb_writel(&rx
->rx_sop
, 0, 0);
89 musb_writel(&rx
->rx_current
, 0, 0);
90 musb_writel(&rx
->rx_buf_current
, 0, 0);
91 musb_writel(&rx
->rx_len_len
, 0, 0);
92 musb_writel(&rx
->rx_cnt_cnt
, 0, 0);
95 /* zero out entire tx state RAM entry for the channel */
96 static void cppi_reset_tx(struct cppi_tx_stateram __iomem
*tx
, u32 ptr
)
98 musb_writel(&tx
->tx_head
, 0, 0);
99 musb_writel(&tx
->tx_buf
, 0, 0);
100 musb_writel(&tx
->tx_current
, 0, 0);
101 musb_writel(&tx
->tx_buf_current
, 0, 0);
102 musb_writel(&tx
->tx_info
, 0, 0);
103 musb_writel(&tx
->tx_rem_len
, 0, 0);
104 /* musb_writel(&tx->tx_dummy, 0, 0); */
105 musb_writel(&tx
->tx_complete
, 0, ptr
);
108 static void cppi_pool_init(struct cppi
*cppi
, struct cppi_channel
*c
)
112 /* initialize channel fields */
115 c
->last_processed
= NULL
;
116 c
->channel
.status
= MUSB_DMA_STATUS_UNKNOWN
;
117 c
->controller
= cppi
;
121 /* build the BD Free list for the channel */
122 for (j
= 0; j
< NUM_TXCHAN_BD
+ 1; j
++) {
123 struct cppi_descriptor
*bd
;
126 bd
= dma_pool_alloc(cppi
->pool
, GFP_KERNEL
, &dma
);
132 static int cppi_channel_abort(struct dma_channel
*);
134 static void cppi_pool_free(struct cppi_channel
*c
)
136 struct cppi
*cppi
= c
->controller
;
137 struct cppi_descriptor
*bd
;
139 (void) cppi_channel_abort(&c
->channel
);
140 c
->channel
.status
= MUSB_DMA_STATUS_UNKNOWN
;
141 c
->controller
= NULL
;
143 /* free all its bds */
144 bd
= c
->last_processed
;
147 dma_pool_free(cppi
->pool
, bd
, bd
->dma
);
148 bd
= cppi_bd_alloc(c
);
150 c
->last_processed
= NULL
;
153 static int cppi_controller_start(struct dma_controller
*c
)
155 struct cppi
*controller
;
156 void __iomem
*tibase
;
159 controller
= container_of(c
, struct cppi
, controller
);
161 /* do whatever is necessary to start controller */
162 for (i
= 0; i
< ARRAY_SIZE(controller
->tx
); i
++) {
163 controller
->tx
[i
].transmit
= true;
164 controller
->tx
[i
].index
= i
;
166 for (i
= 0; i
< ARRAY_SIZE(controller
->rx
); i
++) {
167 controller
->rx
[i
].transmit
= false;
168 controller
->rx
[i
].index
= i
;
171 /* setup BD list on a per channel basis */
172 for (i
= 0; i
< ARRAY_SIZE(controller
->tx
); i
++)
173 cppi_pool_init(controller
, controller
->tx
+ i
);
174 for (i
= 0; i
< ARRAY_SIZE(controller
->rx
); i
++)
175 cppi_pool_init(controller
, controller
->rx
+ i
);
177 tibase
= controller
->tibase
;
178 INIT_LIST_HEAD(&controller
->tx_complete
);
180 /* initialise tx/rx channel head pointers to zero */
181 for (i
= 0; i
< ARRAY_SIZE(controller
->tx
); i
++) {
182 struct cppi_channel
*tx_ch
= controller
->tx
+ i
;
183 struct cppi_tx_stateram __iomem
*tx
;
185 INIT_LIST_HEAD(&tx_ch
->tx_complete
);
187 tx
= tibase
+ DAVINCI_TXCPPI_STATERAM_OFFSET(i
);
188 tx_ch
->state_ram
= tx
;
189 cppi_reset_tx(tx
, 0);
191 for (i
= 0; i
< ARRAY_SIZE(controller
->rx
); i
++) {
192 struct cppi_channel
*rx_ch
= controller
->rx
+ i
;
193 struct cppi_rx_stateram __iomem
*rx
;
195 INIT_LIST_HEAD(&rx_ch
->tx_complete
);
197 rx
= tibase
+ DAVINCI_RXCPPI_STATERAM_OFFSET(i
);
198 rx_ch
->state_ram
= rx
;
202 /* enable individual cppi channels */
203 musb_writel(tibase
, DAVINCI_TXCPPI_INTENAB_REG
,
204 DAVINCI_DMA_ALL_CHANNELS_ENABLE
);
205 musb_writel(tibase
, DAVINCI_RXCPPI_INTENAB_REG
,
206 DAVINCI_DMA_ALL_CHANNELS_ENABLE
);
208 /* enable tx/rx CPPI control */
209 musb_writel(tibase
, DAVINCI_TXCPPI_CTRL_REG
, DAVINCI_DMA_CTRL_ENABLE
);
210 musb_writel(tibase
, DAVINCI_RXCPPI_CTRL_REG
, DAVINCI_DMA_CTRL_ENABLE
);
212 /* disable RNDIS mode, also host rx RNDIS autorequest */
213 musb_writel(tibase
, DAVINCI_RNDIS_REG
, 0);
214 musb_writel(tibase
, DAVINCI_AUTOREQ_REG
, 0);
220 * Stop DMA controller
222 * De-Init the DMA controller as necessary.
225 static int cppi_controller_stop(struct dma_controller
*c
)
227 struct cppi
*controller
;
228 void __iomem
*tibase
;
232 controller
= container_of(c
, struct cppi
, controller
);
233 musb
= controller
->musb
;
235 tibase
= controller
->tibase
;
236 /* DISABLE INDIVIDUAL CHANNEL Interrupts */
237 musb_writel(tibase
, DAVINCI_TXCPPI_INTCLR_REG
,
238 DAVINCI_DMA_ALL_CHANNELS_ENABLE
);
239 musb_writel(tibase
, DAVINCI_RXCPPI_INTCLR_REG
,
240 DAVINCI_DMA_ALL_CHANNELS_ENABLE
);
242 dev_dbg(musb
->controller
, "Tearing down RX and TX Channels\n");
243 for (i
= 0; i
< ARRAY_SIZE(controller
->tx
); i
++) {
244 /* FIXME restructure of txdma to use bds like rxdma */
245 controller
->tx
[i
].last_processed
= NULL
;
246 cppi_pool_free(controller
->tx
+ i
);
248 for (i
= 0; i
< ARRAY_SIZE(controller
->rx
); i
++)
249 cppi_pool_free(controller
->rx
+ i
);
251 /* in Tx Case proper teardown is supported. We resort to disabling
252 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
253 * complete TX CPPI cannot be disabled.
255 /*disable tx/rx cppi */
256 musb_writel(tibase
, DAVINCI_TXCPPI_CTRL_REG
, DAVINCI_DMA_CTRL_DISABLE
);
257 musb_writel(tibase
, DAVINCI_RXCPPI_CTRL_REG
, DAVINCI_DMA_CTRL_DISABLE
);
262 /* While dma channel is allocated, we only want the core irqs active
263 * for fault reports, otherwise we'd get irqs that we don't care about.
264 * Except for TX irqs, where dma done != fifo empty and reusable ...
266 * NOTE: docs don't say either way, but irq masking **enables** irqs.
268 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
270 static inline void core_rxirq_disable(void __iomem
*tibase
, unsigned epnum
)
272 musb_writel(tibase
, DAVINCI_USB_INT_MASK_CLR_REG
, 1 << (epnum
+ 8));
275 static inline void core_rxirq_enable(void __iomem
*tibase
, unsigned epnum
)
277 musb_writel(tibase
, DAVINCI_USB_INT_MASK_SET_REG
, 1 << (epnum
+ 8));
282 * Allocate a CPPI Channel for DMA. With CPPI, channels are bound to
283 * each transfer direction of a non-control endpoint, so allocating
284 * (and deallocating) is mostly a way to notice bad housekeeping on
285 * the software side. We assume the irqs are always active.
287 static struct dma_channel
*
288 cppi_channel_allocate(struct dma_controller
*c
,
289 struct musb_hw_ep
*ep
, u8 transmit
)
291 struct cppi
*controller
;
293 struct cppi_channel
*cppi_ch
;
294 void __iomem
*tibase
;
297 controller
= container_of(c
, struct cppi
, controller
);
298 tibase
= controller
->tibase
;
299 musb
= controller
->musb
;
301 /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
302 index
= ep
->epnum
- 1;
304 /* return the corresponding CPPI Channel Handle, and
305 * probably disable the non-CPPI irq until we need it.
308 if (index
>= ARRAY_SIZE(controller
->tx
)) {
309 dev_dbg(musb
->controller
, "no %cX%d CPPI channel\n", 'T', index
);
312 cppi_ch
= controller
->tx
+ index
;
314 if (index
>= ARRAY_SIZE(controller
->rx
)) {
315 dev_dbg(musb
->controller
, "no %cX%d CPPI channel\n", 'R', index
);
318 cppi_ch
= controller
->rx
+ index
;
319 core_rxirq_disable(tibase
, ep
->epnum
);
322 /* REVISIT make this an error later once the same driver code works
323 * with the other DMA engine too
326 dev_dbg(musb
->controller
, "re-allocating DMA%d %cX channel %p\n",
327 index
, transmit
? 'T' : 'R', cppi_ch
);
329 cppi_ch
->channel
.status
= MUSB_DMA_STATUS_FREE
;
330 cppi_ch
->channel
.max_len
= 0x7fffffff;
332 dev_dbg(musb
->controller
, "Allocate CPPI%d %cX\n", index
, transmit
? 'T' : 'R');
333 return &cppi_ch
->channel
;
336 /* Release a CPPI Channel. */
337 static void cppi_channel_release(struct dma_channel
*channel
)
339 struct cppi_channel
*c
;
340 void __iomem
*tibase
;
342 /* REVISIT: for paranoia, check state and abort if needed... */
344 c
= container_of(channel
, struct cppi_channel
, channel
);
345 tibase
= c
->controller
->tibase
;
347 dev_dbg(c
->controller
->musb
->controller
,
348 "releasing idle DMA channel %p\n", c
);
349 else if (!c
->transmit
)
350 core_rxirq_enable(tibase
, c
->index
+ 1);
352 /* for now, leave its cppi IRQ enabled (we won't trigger it) */
354 channel
->status
= MUSB_DMA_STATUS_UNKNOWN
;
357 /* Context: controller irqlocked */
359 cppi_dump_rx(int level
, struct cppi_channel
*c
, const char *tag
)
361 void __iomem
*base
= c
->controller
->mregs
;
362 struct cppi_rx_stateram __iomem
*rx
= c
->state_ram
;
364 musb_ep_select(base
, c
->index
+ 1);
366 dev_dbg(c
->controller
->musb
->controller
,
367 "RX DMA%d%s: %d left, csr %04x, "
368 "%08x H%08x S%08x C%08x, "
369 "B%08x L%08x %08x .. %08x"
372 musb_readl(c
->controller
->tibase
,
373 DAVINCI_RXCPPI_BUFCNT0_REG
+ 4 * c
->index
),
374 musb_readw(c
->hw_ep
->regs
, MUSB_RXCSR
),
376 musb_readl(&rx
->rx_skipbytes
, 0),
377 musb_readl(&rx
->rx_head
, 0),
378 musb_readl(&rx
->rx_sop
, 0),
379 musb_readl(&rx
->rx_current
, 0),
381 musb_readl(&rx
->rx_buf_current
, 0),
382 musb_readl(&rx
->rx_len_len
, 0),
383 musb_readl(&rx
->rx_cnt_cnt
, 0),
384 musb_readl(&rx
->rx_complete
, 0)
388 /* Context: controller irqlocked */
390 cppi_dump_tx(int level
, struct cppi_channel
*c
, const char *tag
)
392 void __iomem
*base
= c
->controller
->mregs
;
393 struct cppi_tx_stateram __iomem
*tx
= c
->state_ram
;
395 musb_ep_select(base
, c
->index
+ 1);
397 dev_dbg(c
->controller
->musb
->controller
,
398 "TX DMA%d%s: csr %04x, "
399 "H%08x S%08x C%08x %08x, "
400 "F%08x L%08x .. %08x"
403 musb_readw(c
->hw_ep
->regs
, MUSB_TXCSR
),
405 musb_readl(&tx
->tx_head
, 0),
406 musb_readl(&tx
->tx_buf
, 0),
407 musb_readl(&tx
->tx_current
, 0),
408 musb_readl(&tx
->tx_buf_current
, 0),
410 musb_readl(&tx
->tx_info
, 0),
411 musb_readl(&tx
->tx_rem_len
, 0),
412 /* dummy/unused word 6 */
413 musb_readl(&tx
->tx_complete
, 0)
417 /* Context: controller irqlocked */
419 cppi_rndis_update(struct cppi_channel
*c
, int is_rx
,
420 void __iomem
*tibase
, int is_rndis
)
422 /* we may need to change the rndis flag for this cppi channel */
423 if (c
->is_rndis
!= is_rndis
) {
424 u32 value
= musb_readl(tibase
, DAVINCI_RNDIS_REG
);
425 u32 temp
= 1 << (c
->index
);
433 musb_writel(tibase
, DAVINCI_RNDIS_REG
, value
);
434 c
->is_rndis
= is_rndis
;
438 #ifdef CONFIG_USB_MUSB_DEBUG
439 static void cppi_dump_rxbd(const char *tag
, struct cppi_descriptor
*bd
)
441 pr_debug("RXBD/%s %08x: "
442 "nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
444 bd
->hw_next
, bd
->hw_bufp
, bd
->hw_off_len
,
449 static void cppi_dump_rxq(int level
, const char *tag
, struct cppi_channel
*rx
)
451 #ifdef CONFIG_USB_MUSB_DEBUG
452 struct cppi_descriptor
*bd
;
454 if (!_dbg_level(level
))
456 cppi_dump_rx(level
, rx
, tag
);
457 if (rx
->last_processed
)
458 cppi_dump_rxbd("last", rx
->last_processed
);
459 for (bd
= rx
->head
; bd
; bd
= bd
->next
)
460 cppi_dump_rxbd("active", bd
);
465 /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
466 * so we won't ever use it (see "CPPI RX Woes" below).
468 static inline int cppi_autoreq_update(struct cppi_channel
*rx
,
469 void __iomem
*tibase
, int onepacket
, unsigned n_bds
)
473 #ifdef RNDIS_RX_IS_USABLE
475 /* assert(is_host_active(musb)) */
477 /* start from "AutoReq never" */
478 tmp
= musb_readl(tibase
, DAVINCI_AUTOREQ_REG
);
479 val
= tmp
& ~((0x3) << (rx
->index
* 2));
481 /* HCD arranged reqpkt for packet #1. we arrange int
482 * for all but the last one, maybe in two segments.
486 /* use two segments, autoreq "all" then the last "never" */
487 val
|= ((0x3) << (rx
->index
* 2));
490 /* one segment, autoreq "all-but-last" */
491 val
|= ((0x1) << (rx
->index
* 2));
498 /* make sure that autoreq is updated before continuing */
499 musb_writel(tibase
, DAVINCI_AUTOREQ_REG
, val
);
501 tmp
= musb_readl(tibase
, DAVINCI_AUTOREQ_REG
);
509 /* REQPKT is turned off after each segment */
510 if (n_bds
&& rx
->channel
.actual_len
) {
511 void __iomem
*regs
= rx
->hw_ep
->regs
;
513 val
= musb_readw(regs
, MUSB_RXCSR
);
514 if (!(val
& MUSB_RXCSR_H_REQPKT
)) {
515 val
|= MUSB_RXCSR_H_REQPKT
| MUSB_RXCSR_H_WZC_BITS
;
516 musb_writew(regs
, MUSB_RXCSR
, val
);
517 /* flush writebuffer */
518 val
= musb_readw(regs
, MUSB_RXCSR
);
525 /* Buffer enqueuing Logic:
527 * - RX builds new queues each time, to help handle routine "early
528 * termination" cases (faults, including errors and short reads)
531 * - for now, TX reuses the same queue of BDs every time
533 * REVISIT long term, we want a normal dynamic model.
534 * ... the goal will be to append to the
535 * existing queue, processing completed "dma buffers" (segments) on the fly.
537 * Otherwise we force an IRQ latency between requests, which slows us a lot
538 * (especially in "transparent" dma). Unfortunately that model seems to be
539 * inherent in the DMA model from the Mentor code, except in the rare case
540 * of transfers big enough (~128+ KB) that we could append "middle" segments
541 * in the TX paths. (RX can't do this, see below.)
543 * That's true even in the CPPI- friendly iso case, where most urbs have
544 * several small segments provided in a group and where the "packet at a time"
545 * "transparent" DMA model is always correct, even on the RX side.
551 * TX is a lot more reasonable than RX; it doesn't need to run in
552 * irq-per-packet mode very often. RNDIS mode seems to behave too
553 * (except how it handles the exactly-N-packets case). Building a
554 * txdma queue with multiple requests (urb or usb_request) looks
555 * like it would work ... but fault handling would need much testing.
557 * The main issue with TX mode RNDIS relates to transfer lengths that
558 * are an exact multiple of the packet length. It appears that there's
559 * a hiccup in that case (maybe the DMA completes before the ZLP gets
560 * written?) boiling down to not being able to rely on CPPI writing any
561 * terminating zero length packet before the next transfer is written.
562 * So that's punted to PIO; better yet, gadget drivers can avoid it.
564 * Plus, there's allegedly an undocumented constraint that rndis transfer
565 * length be a multiple of 64 bytes ... but the chip doesn't act that
566 * way, and we really don't _want_ that behavior anyway.
568 * On TX, "transparent" mode works ... although experiments have shown
569 * problems trying to use the SOP/EOP bits in different USB packets.
571 * REVISIT try to handle terminating zero length packets using CPPI
572 * instead of doing it by PIO after an IRQ. (Meanwhile, make Ethernet
573 * links avoid that issue by forcing them to avoid zlps.)
576 cppi_next_tx_segment(struct musb
*musb
, struct cppi_channel
*tx
)
578 unsigned maxpacket
= tx
->maxpacket
;
579 dma_addr_t addr
= tx
->buf_dma
+ tx
->offset
;
580 size_t length
= tx
->buf_len
- tx
->offset
;
581 struct cppi_descriptor
*bd
;
584 struct cppi_tx_stateram __iomem
*tx_ram
= tx
->state_ram
;
587 /* TX can use the CPPI "rndis" mode, where we can probably fit this
588 * transfer in one BD and one IRQ. The only time we would NOT want
589 * to use it is when hardware constraints prevent it, or if we'd
590 * trigger the "send a ZLP?" confusion.
592 rndis
= (maxpacket
& 0x3f) == 0
593 && length
> maxpacket
595 && (length
% maxpacket
) != 0;
601 n_bds
= length
/ maxpacket
;
602 if (!length
|| (length
% maxpacket
))
604 n_bds
= min(n_bds
, (unsigned) NUM_TXCHAN_BD
);
605 length
= min(n_bds
* maxpacket
, length
);
608 dev_dbg(musb
->controller
, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
611 rndis
? "rndis" : "transparent",
613 (unsigned long long)addr
, length
);
615 cppi_rndis_update(tx
, 0, musb
->ctrl_base
, rndis
);
617 /* assuming here that channel_program is called during
618 * transfer initiation ... current code maintains state
619 * for one outstanding request only (no queues, not even
620 * the implicit ones of an iso urb).
625 tx
->last_processed
= NULL
;
627 /* FIXME use BD pool like RX side does, and just queue
628 * the minimum number for this request.
631 /* Prepare queue of BDs first, then hand it to hardware.
632 * All BDs except maybe the last should be of full packet
633 * size; for RNDIS there _is_ only that last packet.
635 for (i
= 0; i
< n_bds
; ) {
636 if (++i
< n_bds
&& bd
->next
)
637 bd
->hw_next
= bd
->next
->dma
;
641 bd
->hw_bufp
= tx
->buf_dma
+ tx
->offset
;
643 /* FIXME set EOP only on the last packet,
644 * SOP only on the first ... avoid IRQs
646 if ((tx
->offset
+ maxpacket
) <= tx
->buf_len
) {
647 tx
->offset
+= maxpacket
;
648 bd
->hw_off_len
= maxpacket
;
649 bd
->hw_options
= CPPI_SOP_SET
| CPPI_EOP_SET
650 | CPPI_OWN_SET
| maxpacket
;
652 /* only this one may be a partial USB Packet */
655 partial_len
= tx
->buf_len
- tx
->offset
;
656 tx
->offset
= tx
->buf_len
;
657 bd
->hw_off_len
= partial_len
;
659 bd
->hw_options
= CPPI_SOP_SET
| CPPI_EOP_SET
660 | CPPI_OWN_SET
| partial_len
;
661 if (partial_len
== 0)
662 bd
->hw_options
|= CPPI_ZERO_SET
;
665 dev_dbg(musb
->controller
, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
666 bd
, bd
->hw_next
, bd
->hw_bufp
,
667 bd
->hw_off_len
, bd
->hw_options
);
669 /* update the last BD enqueued to the list */
674 /* BDs live in DMA-coherent memory, but writes might be pending */
675 cpu_drain_writebuffer();
677 /* Write to the HeadPtr in state RAM to trigger */
678 musb_writel(&tx_ram
->tx_head
, 0, (u32
)tx
->freelist
->dma
);
680 cppi_dump_tx(5, tx
, "/S");
686 * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
687 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
688 * (Full speed transfers have similar scenarios.)
690 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
691 * and the next packet goes into a buffer that's queued later; while (b) fills
692 * the buffer with 1024 bytes. How to do that with CPPI?
694 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
695 * (b) loses **BADLY** because nothing (!) happens when that second packet
696 * fills the buffer, much less when a third one arrives. (Which makes this
697 * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
698 * is optional, and it's fine if peripherals -- not hosts! -- pad messages
699 * out to end-of-buffer. Standard PCI host controller DMA descriptors
700 * implement that mode by default ... which is no accident.)
702 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
703 * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
704 * ignores SOP/EOP markings and processes both of those BDs; so both packets
705 * are loaded into the buffer (with a 212 byte gap between them), and the next
706 * buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
707 * are intended as outputs for RX queues, not inputs...)
709 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
710 * reliably make both cases work, with software handling both cases correctly
711 * and at the significant penalty of needing an IRQ per packet. (The lack of
712 * I/O overlap can be slightly ameliorated by enabling double buffering.)
714 * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
715 * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
716 * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
717 * with guaranteed driver level fault recovery and scrubbing out what's left
718 * of that garbaged datastream.
720 * But there seems to be no way to identify the cases where CPPI RNDIS mode
721 * is appropriate -- which do NOT include RNDIS host drivers, but do include
722 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
723 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
724 * that applies best on the peripheral side (and which could fail rudely).
726 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
727 * cases other than mass storage class. Otherwise we're correct but slow,
728 * since CPPI penalizes our need for a "true RNDIS" default mode.
732 /* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
735 * (a) peripheral mode ... since rndis peripherals could pad their
736 * writes to hosts, causing i/o failure; or we'd have to cope with
737 * a largely unknowable variety of host side protocol variants
738 * (b) and short reads are NOT errors ... since full reads would
739 * cause those same i/o failures
740 * (c) and read length is
741 * - less than 64KB (max per cppi descriptor)
742 * - not a multiple of 4096 (g_zero default, full reads typical)
743 * - N (>1) packets long, ditto (full reads not EXPECTED)
747 * Cost of heuristic failing: RXDMA wedges at the end of transfers that
748 * fill out the whole buffer. Buggy host side usb network drivers could
749 * trigger that, but "in the field" such bugs seem to be all but unknown.
751 * So this module parameter lets the heuristic be disabled. When using
752 * gadgetfs, the heuristic will probably need to be disabled.
754 static bool cppi_rx_rndis
= 1;
756 module_param(cppi_rx_rndis
, bool, 0);
757 MODULE_PARM_DESC(cppi_rx_rndis
, "enable/disable RX RNDIS heuristic");
761 * cppi_next_rx_segment - dma read for the next chunk of a buffer
762 * @musb: the controller
764 * @onepacket: true unless caller treats short reads as errors, and
765 * performs fault recovery above usbcore.
766 * Context: controller irqlocked
768 * See above notes about why we can't use multi-BD RX queues except in
769 * rare cases (mass storage class), and can never use the hardware "rndis"
770 * mode (since it's not a "true" RNDIS mode) with complete safety..
772 * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
773 * code to recover from corrupted datastreams after each short transfer.
776 cppi_next_rx_segment(struct musb
*musb
, struct cppi_channel
*rx
, int onepacket
)
778 unsigned maxpacket
= rx
->maxpacket
;
779 dma_addr_t addr
= rx
->buf_dma
+ rx
->offset
;
780 size_t length
= rx
->buf_len
- rx
->offset
;
781 struct cppi_descriptor
*bd
, *tail
;
784 void __iomem
*tibase
= musb
->ctrl_base
;
786 struct cppi_rx_stateram __iomem
*rx_ram
= rx
->state_ram
;
789 /* almost every USB driver, host or peripheral side */
792 /* maybe apply the heuristic above */
794 && is_peripheral_active(musb
)
795 && length
> maxpacket
796 && (length
& ~0xffff) == 0
797 && (length
& 0x0fff) != 0
798 && (length
& (maxpacket
- 1)) == 0) {
803 /* virtually nothing except mass storage class */
804 if (length
> 0xffff) {
805 n_bds
= 0xffff / maxpacket
;
806 length
= n_bds
* maxpacket
;
808 n_bds
= length
/ maxpacket
;
809 if (length
% maxpacket
)
815 n_bds
= min(n_bds
, (unsigned) NUM_RXCHAN_BD
);
818 /* In host mode, autorequest logic can generate some IN tokens; it's
819 * tricky since we can't leave REQPKT set in RXCSR after the transfer
820 * finishes. So: multipacket transfers involve two or more segments.
821 * And always at least two IRQs ... RNDIS mode is not an option.
823 if (is_host_active(musb
))
824 n_bds
= cppi_autoreq_update(rx
, tibase
, onepacket
, n_bds
);
826 cppi_rndis_update(rx
, 1, musb
->ctrl_base
, is_rndis
);
828 length
= min(n_bds
* maxpacket
, length
);
830 dev_dbg(musb
->controller
, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
831 "dma 0x%llx len %u %u/%u\n",
832 rx
->index
, maxpacket
,
834 ? (is_rndis
? "rndis" : "onepacket")
838 DAVINCI_RXCPPI_BUFCNT0_REG
+ (rx
->index
* 4))
840 (unsigned long long)addr
, length
,
841 rx
->channel
.actual_len
, rx
->buf_len
);
843 /* only queue one segment at a time, since the hardware prevents
844 * correct queue shutdown after unexpected short packets
846 bd
= cppi_bd_alloc(rx
);
849 /* Build BDs for all packets in this segment */
850 for (i
= 0, tail
= NULL
; bd
&& i
< n_bds
; i
++, tail
= bd
) {
854 bd
= cppi_bd_alloc(rx
);
858 tail
->hw_next
= bd
->dma
;
862 /* all but the last packet will be maxpacket size */
863 if (maxpacket
< length
)
870 rx
->offset
+= bd_len
;
872 bd
->hw_off_len
= (0 /*offset*/ << 16) + bd_len
;
875 bd
->hw_options
= CPPI_OWN_SET
| (i
== 0 ? length
: 0);
879 /* we always expect at least one reusable BD! */
881 WARNING("rx dma%d -- no BDs? need %d\n", rx
->index
, n_bds
);
883 } else if (i
< n_bds
)
884 WARNING("rx dma%d -- only %d of %d BDs\n", rx
->index
, i
, n_bds
);
892 /* short reads and other faults should terminate this entire
893 * dma segment. we want one "dma packet" per dma segment, not
894 * one per USB packet, terminating the whole queue at once...
895 * NOTE that current hardware seems to ignore SOP and EOP.
897 bd
->hw_options
|= CPPI_SOP_SET
;
898 tail
->hw_options
|= CPPI_EOP_SET
;
900 #ifdef CONFIG_USB_MUSB_DEBUG
902 struct cppi_descriptor
*d
;
904 for (d
= rx
->head
; d
; d
= d
->next
)
905 cppi_dump_rxbd("S", d
);
909 /* in case the preceding transfer left some state... */
910 tail
= rx
->last_processed
;
913 tail
->hw_next
= bd
->dma
;
916 core_rxirq_enable(tibase
, rx
->index
+ 1);
918 /* BDs live in DMA-coherent memory, but writes might be pending */
919 cpu_drain_writebuffer();
921 /* REVISIT specs say to write this AFTER the BUFCNT register
922 * below ... but that loses badly.
924 musb_writel(&rx_ram
->rx_head
, 0, bd
->dma
);
926 /* bufferCount must be at least 3, and zeroes on completion
927 * unless it underflows below zero, or stops at two, or keeps
930 i
= musb_readl(tibase
,
931 DAVINCI_RXCPPI_BUFCNT0_REG
+ (rx
->index
* 4))
936 DAVINCI_RXCPPI_BUFCNT0_REG
+ (rx
->index
* 4),
938 else if (n_bds
> (i
- 3))
940 DAVINCI_RXCPPI_BUFCNT0_REG
+ (rx
->index
* 4),
943 i
= musb_readl(tibase
,
944 DAVINCI_RXCPPI_BUFCNT0_REG
+ (rx
->index
* 4))
946 if (i
< (2 + n_bds
)) {
947 dev_dbg(musb
->controller
, "bufcnt%d underrun - %d (for %d)\n",
948 rx
->index
, i
, n_bds
);
950 DAVINCI_RXCPPI_BUFCNT0_REG
+ (rx
->index
* 4),
954 cppi_dump_rx(4, rx
, "/S");
958 * cppi_channel_program - program channel for data transfer
960 * @maxpacket: max packet size
961 * @mode: For RX, 1 unless the usb protocol driver promised to treat
962 * all short reads as errors and kick in high level fault recovery.
963 * For TX, ignored because of RNDIS mode races/glitches.
964 * @dma_addr: dma address of buffer
965 * @len: length of buffer
966 * Context: controller irqlocked
968 static int cppi_channel_program(struct dma_channel
*ch
,
969 u16 maxpacket
, u8 mode
,
970 dma_addr_t dma_addr
, u32 len
)
972 struct cppi_channel
*cppi_ch
;
973 struct cppi
*controller
;
976 cppi_ch
= container_of(ch
, struct cppi_channel
, channel
);
977 controller
= cppi_ch
->controller
;
978 musb
= controller
->musb
;
980 switch (ch
->status
) {
981 case MUSB_DMA_STATUS_BUS_ABORT
:
982 case MUSB_DMA_STATUS_CORE_ABORT
:
983 /* fault irq handler should have handled cleanup */
984 WARNING("%cX DMA%d not cleaned up after abort!\n",
985 cppi_ch
->transmit
? 'T' : 'R',
989 case MUSB_DMA_STATUS_BUSY
:
990 WARNING("program active channel? %cX DMA%d\n",
991 cppi_ch
->transmit
? 'T' : 'R',
995 case MUSB_DMA_STATUS_UNKNOWN
:
996 dev_dbg(musb
->controller
, "%cX DMA%d not allocated!\n",
997 cppi_ch
->transmit
? 'T' : 'R',
1000 case MUSB_DMA_STATUS_FREE
:
1004 ch
->status
= MUSB_DMA_STATUS_BUSY
;
1006 /* set transfer parameters, then queue up its first segment */
1007 cppi_ch
->buf_dma
= dma_addr
;
1008 cppi_ch
->offset
= 0;
1009 cppi_ch
->maxpacket
= maxpacket
;
1010 cppi_ch
->buf_len
= len
;
1011 cppi_ch
->channel
.actual_len
= 0;
1013 /* TX channel? or RX? */
1014 if (cppi_ch
->transmit
)
1015 cppi_next_tx_segment(musb
, cppi_ch
);
1017 cppi_next_rx_segment(musb
, cppi_ch
, mode
);
1022 static bool cppi_rx_scan(struct cppi
*cppi
, unsigned ch
)
1024 struct cppi_channel
*rx
= &cppi
->rx
[ch
];
1025 struct cppi_rx_stateram __iomem
*state
= rx
->state_ram
;
1026 struct cppi_descriptor
*bd
;
1027 struct cppi_descriptor
*last
= rx
->last_processed
;
1028 bool completed
= false;
1031 dma_addr_t safe2ack
;
1032 void __iomem
*regs
= rx
->hw_ep
->regs
;
1033 struct musb
*musb
= cppi
->musb
;
1035 cppi_dump_rx(6, rx
, "/K");
1037 bd
= last
? last
->next
: rx
->head
;
1041 /* run through all completed BDs */
1042 for (i
= 0, safe2ack
= musb_readl(&state
->rx_complete
, 0);
1043 (safe2ack
|| completed
) && bd
&& i
< NUM_RXCHAN_BD
;
1044 i
++, bd
= bd
->next
) {
1047 /* catch latest BD writes from CPPI */
1049 if (!completed
&& (bd
->hw_options
& CPPI_OWN_SET
))
1052 dev_dbg(musb
->controller
, "C/RXBD %llx: nxt %08x buf %08x "
1053 "off.len %08x opt.len %08x (%d)\n",
1054 (unsigned long long)bd
->dma
, bd
->hw_next
, bd
->hw_bufp
,
1055 bd
->hw_off_len
, bd
->hw_options
,
1056 rx
->channel
.actual_len
);
1058 /* actual packet received length */
1059 if ((bd
->hw_options
& CPPI_SOP_SET
) && !completed
)
1060 len
= bd
->hw_off_len
& CPPI_RECV_PKTLEN_MASK
;
1064 if (bd
->hw_options
& CPPI_EOQ_MASK
)
1067 if (!completed
&& len
< bd
->buflen
) {
1068 /* NOTE: when we get a short packet, RXCSR_H_REQPKT
1069 * must have been cleared, and no more DMA packets may
1070 * active be in the queue... TI docs didn't say, but
1071 * CPPI ignores those BDs even though OWN is still set.
1074 dev_dbg(musb
->controller
, "rx short %d/%d (%d)\n",
1076 rx
->channel
.actual_len
);
1079 /* If we got here, we expect to ack at least one BD; meanwhile
1080 * CPPI may completing other BDs while we scan this list...
1082 * RACE: we can notice OWN cleared before CPPI raises the
1083 * matching irq by writing that BD as the completion pointer.
1084 * In such cases, stop scanning and wait for the irq, avoiding
1085 * lost acks and states where BD ownership is unclear.
1087 if (bd
->dma
== safe2ack
) {
1088 musb_writel(&state
->rx_complete
, 0, safe2ack
);
1089 safe2ack
= musb_readl(&state
->rx_complete
, 0);
1091 if (bd
->dma
== safe2ack
)
1095 rx
->channel
.actual_len
+= len
;
1097 cppi_bd_free(rx
, last
);
1100 /* stop scanning on end-of-segment */
1101 if (bd
->hw_next
== 0)
1104 rx
->last_processed
= last
;
1106 /* dma abort, lost ack, or ... */
1107 if (!acked
&& last
) {
1110 if (safe2ack
== 0 || safe2ack
== rx
->last_processed
->dma
)
1111 musb_writel(&state
->rx_complete
, 0, safe2ack
);
1112 if (safe2ack
== 0) {
1113 cppi_bd_free(rx
, last
);
1114 rx
->last_processed
= NULL
;
1116 /* if we land here on the host side, H_REQPKT will
1117 * be clear and we need to restart the queue...
1121 musb_ep_select(cppi
->mregs
, rx
->index
+ 1);
1122 csr
= musb_readw(regs
, MUSB_RXCSR
);
1123 if (csr
& MUSB_RXCSR_DMAENAB
) {
1124 dev_dbg(musb
->controller
, "list%d %p/%p, last %llx%s, csr %04x\n",
1128 ? (unsigned long long)
1129 rx
->last_processed
->dma
1131 completed
? ", completed" : "",
1133 cppi_dump_rxq(4, "/what?", rx
);
1141 /* REVISIT seems like "autoreq all but EOP" doesn't...
1142 * setting it here "should" be racey, but seems to work
1144 csr
= musb_readw(rx
->hw_ep
->regs
, MUSB_RXCSR
);
1145 if (is_host_active(cppi
->musb
)
1147 && !(csr
& MUSB_RXCSR_H_REQPKT
)) {
1148 csr
|= MUSB_RXCSR_H_REQPKT
;
1149 musb_writew(regs
, MUSB_RXCSR
,
1150 MUSB_RXCSR_H_WZC_BITS
| csr
);
1151 csr
= musb_readw(rx
->hw_ep
->regs
, MUSB_RXCSR
);
1158 cppi_dump_rx(6, rx
, completed
? "/completed" : "/cleaned");
1162 irqreturn_t
cppi_interrupt(int irq
, void *dev_id
)
1164 struct musb
*musb
= dev_id
;
1166 void __iomem
*tibase
;
1167 struct musb_hw_ep
*hw_ep
= NULL
;
1170 unsigned long uninitialized_var(flags
);
1172 cppi
= container_of(musb
->dma_controller
, struct cppi
, controller
);
1174 spin_lock_irqsave(&musb
->lock
, flags
);
1176 tibase
= musb
->ctrl_base
;
1178 tx
= musb_readl(tibase
, DAVINCI_TXCPPI_MASKED_REG
);
1179 rx
= musb_readl(tibase
, DAVINCI_RXCPPI_MASKED_REG
);
1183 spin_unlock_irqrestore(&musb
->lock
, flags
);
1187 dev_dbg(musb
->controller
, "CPPI IRQ Tx%x Rx%x\n", tx
, rx
);
1189 /* process TX channels */
1190 for (index
= 0; tx
; tx
= tx
>> 1, index
++) {
1191 struct cppi_channel
*tx_ch
;
1192 struct cppi_tx_stateram __iomem
*tx_ram
;
1193 bool completed
= false;
1194 struct cppi_descriptor
*bd
;
1199 tx_ch
= cppi
->tx
+ index
;
1200 tx_ram
= tx_ch
->state_ram
;
1202 /* FIXME need a cppi_tx_scan() routine, which
1203 * can also be called from abort code
1206 cppi_dump_tx(5, tx_ch
, "/E");
1211 * If Head is null then this could mean that a abort interrupt
1212 * that needs to be acknowledged.
1215 dev_dbg(musb
->controller
, "null BD\n");
1216 musb_writel(&tx_ram
->tx_complete
, 0, 0);
1220 /* run through all completed BDs */
1221 for (i
= 0; !completed
&& bd
&& i
< NUM_TXCHAN_BD
;
1222 i
++, bd
= bd
->next
) {
1225 /* catch latest BD writes from CPPI */
1227 if (bd
->hw_options
& CPPI_OWN_SET
)
1230 dev_dbg(musb
->controller
, "C/TXBD %p n %x b %x off %x opt %x\n",
1231 bd
, bd
->hw_next
, bd
->hw_bufp
,
1232 bd
->hw_off_len
, bd
->hw_options
);
1234 len
= bd
->hw_off_len
& CPPI_BUFFER_LEN_MASK
;
1235 tx_ch
->channel
.actual_len
+= len
;
1237 tx_ch
->last_processed
= bd
;
1239 /* write completion register to acknowledge
1240 * processing of completed BDs, and possibly
1241 * release the IRQ; EOQ might not be set ...
1243 * REVISIT use the same ack strategy as rx
1245 * REVISIT have observed bit 18 set; huh??
1247 /* if ((bd->hw_options & CPPI_EOQ_MASK)) */
1248 musb_writel(&tx_ram
->tx_complete
, 0, bd
->dma
);
1250 /* stop scanning on end-of-segment */
1251 if (bd
->hw_next
== 0)
1255 /* on end of segment, maybe go to next one */
1257 /* cppi_dump_tx(4, tx_ch, "/complete"); */
1259 /* transfer more, or report completion */
1260 if (tx_ch
->offset
>= tx_ch
->buf_len
) {
1263 tx_ch
->channel
.status
= MUSB_DMA_STATUS_FREE
;
1265 hw_ep
= tx_ch
->hw_ep
;
1267 musb_dma_completion(musb
, index
+ 1, 1);
1270 /* Bigger transfer than we could fit in
1271 * that first batch of descriptors...
1273 cppi_next_tx_segment(musb
, tx_ch
);
1279 /* Start processing the RX block */
1280 for (index
= 0; rx
; rx
= rx
>> 1, index
++) {
1283 struct cppi_channel
*rx_ch
;
1285 rx_ch
= cppi
->rx
+ index
;
1287 /* let incomplete dma segments finish */
1288 if (!cppi_rx_scan(cppi
, index
))
1291 /* start another dma segment if needed */
1292 if (rx_ch
->channel
.actual_len
!= rx_ch
->buf_len
1293 && rx_ch
->channel
.actual_len
1295 cppi_next_rx_segment(musb
, rx_ch
, 1);
1299 /* all segments completed! */
1300 rx_ch
->channel
.status
= MUSB_DMA_STATUS_FREE
;
1302 hw_ep
= rx_ch
->hw_ep
;
1304 core_rxirq_disable(tibase
, index
+ 1);
1305 musb_dma_completion(musb
, index
+ 1, 0);
1309 /* write to CPPI EOI register to re-enable interrupts */
1310 musb_writel(tibase
, DAVINCI_CPPI_EOI_REG
, 0);
1313 spin_unlock_irqrestore(&musb
->lock
, flags
);
1317 EXPORT_SYMBOL_GPL(cppi_interrupt
);
1319 /* Instantiate a software object representing a DMA controller. */
1320 struct dma_controller
*dma_controller_create(struct musb
*musb
, void __iomem
*mregs
)
1322 struct cppi
*controller
;
1323 struct device
*dev
= musb
->controller
;
1324 struct platform_device
*pdev
= to_platform_device(dev
);
1325 int irq
= platform_get_irq_byname(pdev
, "dma");
1327 controller
= kzalloc(sizeof *controller
, GFP_KERNEL
);
1331 controller
->mregs
= mregs
;
1332 controller
->tibase
= mregs
- DAVINCI_BASE_OFFSET
;
1334 controller
->musb
= musb
;
1335 controller
->controller
.start
= cppi_controller_start
;
1336 controller
->controller
.stop
= cppi_controller_stop
;
1337 controller
->controller
.channel_alloc
= cppi_channel_allocate
;
1338 controller
->controller
.channel_release
= cppi_channel_release
;
1339 controller
->controller
.channel_program
= cppi_channel_program
;
1340 controller
->controller
.channel_abort
= cppi_channel_abort
;
1342 /* NOTE: allocating from on-chip SRAM would give the least
1343 * contention for memory access, if that ever matters here.
1346 /* setup BufferPool */
1347 controller
->pool
= dma_pool_create("cppi",
1348 controller
->musb
->controller
,
1349 sizeof(struct cppi_descriptor
),
1350 CPPI_DESCRIPTOR_ALIGN
, 0);
1351 if (!controller
->pool
) {
1357 if (request_irq(irq
, cppi_interrupt
, 0, "cppi-dma", musb
)) {
1358 dev_err(dev
, "request_irq %d failed!\n", irq
);
1359 dma_controller_destroy(&controller
->controller
);
1362 controller
->irq
= irq
;
1365 return &controller
->controller
;
1369 * Destroy a previously-instantiated DMA controller.
1371 void dma_controller_destroy(struct dma_controller
*c
)
1375 cppi
= container_of(c
, struct cppi
, controller
);
1378 free_irq(cppi
->irq
, cppi
->musb
);
1380 /* assert: caller stopped the controller first */
1381 dma_pool_destroy(cppi
->pool
);
1387 * Context: controller irqlocked, endpoint selected
1389 static int cppi_channel_abort(struct dma_channel
*channel
)
1391 struct cppi_channel
*cppi_ch
;
1392 struct cppi
*controller
;
1393 void __iomem
*mbase
;
1394 void __iomem
*tibase
;
1397 struct cppi_descriptor
*queue
;
1399 cppi_ch
= container_of(channel
, struct cppi_channel
, channel
);
1401 controller
= cppi_ch
->controller
;
1403 switch (channel
->status
) {
1404 case MUSB_DMA_STATUS_BUS_ABORT
:
1405 case MUSB_DMA_STATUS_CORE_ABORT
:
1406 /* from RX or TX fault irq handler */
1407 case MUSB_DMA_STATUS_BUSY
:
1408 /* the hardware needs shutting down */
1409 regs
= cppi_ch
->hw_ep
->regs
;
1411 case MUSB_DMA_STATUS_UNKNOWN
:
1412 case MUSB_DMA_STATUS_FREE
:
1418 if (!cppi_ch
->transmit
&& cppi_ch
->head
)
1419 cppi_dump_rxq(3, "/abort", cppi_ch
);
1421 mbase
= controller
->mregs
;
1422 tibase
= controller
->tibase
;
1424 queue
= cppi_ch
->head
;
1425 cppi_ch
->head
= NULL
;
1426 cppi_ch
->tail
= NULL
;
1428 /* REVISIT should rely on caller having done this,
1429 * and caller should rely on us not changing it.
1430 * peripheral code is safe ... check host too.
1432 musb_ep_select(mbase
, cppi_ch
->index
+ 1);
1434 if (cppi_ch
->transmit
) {
1435 struct cppi_tx_stateram __iomem
*tx_ram
;
1436 /* REVISIT put timeouts on these controller handshakes */
1438 cppi_dump_tx(6, cppi_ch
, " (teardown)");
1440 /* teardown DMA engine then usb core */
1442 value
= musb_readl(tibase
, DAVINCI_TXCPPI_TEAR_REG
);
1443 } while (!(value
& CPPI_TEAR_READY
));
1444 musb_writel(tibase
, DAVINCI_TXCPPI_TEAR_REG
, cppi_ch
->index
);
1446 tx_ram
= cppi_ch
->state_ram
;
1448 value
= musb_readl(&tx_ram
->tx_complete
, 0);
1449 } while (0xFFFFFFFC != value
);
1451 /* FIXME clean up the transfer state ... here?
1452 * the completion routine should get called with
1453 * an appropriate status code.
1456 value
= musb_readw(regs
, MUSB_TXCSR
);
1457 value
&= ~MUSB_TXCSR_DMAENAB
;
1458 value
|= MUSB_TXCSR_FLUSHFIFO
;
1459 musb_writew(regs
, MUSB_TXCSR
, value
);
1460 musb_writew(regs
, MUSB_TXCSR
, value
);
1463 * 1. Write to completion Ptr value 0x1(bit 0 set)
1465 * 2. Wait for abort interrupt and then put the channel in
1466 * compare mode by writing 1 to the tx_complete register.
1468 cppi_reset_tx(tx_ram
, 1);
1469 cppi_ch
->head
= NULL
;
1470 musb_writel(&tx_ram
->tx_complete
, 0, 1);
1471 cppi_dump_tx(5, cppi_ch
, " (done teardown)");
1473 /* REVISIT tx side _should_ clean up the same way
1474 * as the RX side ... this does no cleanup at all!
1480 /* NOTE: docs don't guarantee any of this works ... we
1481 * expect that if the usb core stops telling the cppi core
1482 * to pull more data from it, then it'll be safe to flush
1483 * current RX DMA state iff any pending fifo transfer is done.
1486 core_rxirq_disable(tibase
, cppi_ch
->index
+ 1);
1488 /* for host, ensure ReqPkt is never set again */
1489 if (is_host_active(cppi_ch
->controller
->musb
)) {
1490 value
= musb_readl(tibase
, DAVINCI_AUTOREQ_REG
);
1491 value
&= ~((0x3) << (cppi_ch
->index
* 2));
1492 musb_writel(tibase
, DAVINCI_AUTOREQ_REG
, value
);
1495 csr
= musb_readw(regs
, MUSB_RXCSR
);
1497 /* for host, clear (just) ReqPkt at end of current packet(s) */
1498 if (is_host_active(cppi_ch
->controller
->musb
)) {
1499 csr
|= MUSB_RXCSR_H_WZC_BITS
;
1500 csr
&= ~MUSB_RXCSR_H_REQPKT
;
1502 csr
|= MUSB_RXCSR_P_WZC_BITS
;
1504 /* clear dma enable */
1505 csr
&= ~(MUSB_RXCSR_DMAENAB
);
1506 musb_writew(regs
, MUSB_RXCSR
, csr
);
1507 csr
= musb_readw(regs
, MUSB_RXCSR
);
1509 /* Quiesce: wait for current dma to finish (if not cleanup).
1510 * We can't use bit zero of stateram->rx_sop, since that
1511 * refers to an entire "DMA packet" not just emptying the
1512 * current fifo. Most segments need multiple usb packets.
1514 if (channel
->status
== MUSB_DMA_STATUS_BUSY
)
1517 /* scan the current list, reporting any data that was
1518 * transferred and acking any IRQ
1520 cppi_rx_scan(controller
, cppi_ch
->index
);
1522 /* clobber the existing state once it's idle
1524 * NOTE: arguably, we should also wait for all the other
1525 * RX channels to quiesce (how??) and then temporarily
1526 * disable RXCPPI_CTRL_REG ... but it seems that we can
1527 * rely on the controller restarting from state ram, with
1528 * only RXCPPI_BUFCNT state being bogus. BUFCNT will
1529 * correct itself after the next DMA transfer though.
1531 * REVISIT does using rndis mode change that?
1533 cppi_reset_rx(cppi_ch
->state_ram
);
1535 /* next DMA request _should_ load cppi head ptr */
1537 /* ... we don't "free" that list, only mutate it in place. */
1538 cppi_dump_rx(5, cppi_ch
, " (done abort)");
1540 /* clean up previously pending bds */
1541 cppi_bd_free(cppi_ch
, cppi_ch
->last_processed
);
1542 cppi_ch
->last_processed
= NULL
;
1545 struct cppi_descriptor
*tmp
= queue
->next
;
1547 cppi_bd_free(cppi_ch
, queue
);
1552 channel
->status
= MUSB_DMA_STATUS_FREE
;
1553 cppi_ch
->buf_dma
= 0;
1554 cppi_ch
->offset
= 0;
1555 cppi_ch
->buf_len
= 0;
1556 cppi_ch
->maxpacket
= 0;
1562 * Power Management ... probably turn off cppi during suspend, restart;
1563 * check state ram? Clocking is presumably shared with usb core.