1 // SPDX-License-Identifier: GPL-2.0-only
3 * Special handling for DW DMA core
5 * Copyright (c) 2009, 2014 Intel Corporation.
8 #include <linux/completion.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/dmaengine.h>
11 #include <linux/irqreturn.h>
12 #include <linux/jiffies.h>
13 #include <linux/module.h>
14 #include <linux/pci.h>
15 #include <linux/platform_data/dma-dw.h>
16 #include <linux/spi/spi.h>
17 #include <linux/types.h>
21 #define DW_SPI_RX_BUSY 0
22 #define DW_SPI_RX_BURST_LEVEL 16
23 #define DW_SPI_TX_BUSY 1
24 #define DW_SPI_TX_BURST_LEVEL 16
26 static bool dw_spi_dma_chan_filter(struct dma_chan
*chan
, void *param
)
28 struct dw_dma_slave
*s
= param
;
30 if (s
->dma_dev
!= chan
->device
->dev
)
37 static void dw_spi_dma_maxburst_init(struct dw_spi
*dws
)
39 struct dma_slave_caps caps
;
40 u32 max_burst
, def_burst
;
43 def_burst
= dws
->fifo_len
/ 2;
45 ret
= dma_get_slave_caps(dws
->rxchan
, &caps
);
46 if (!ret
&& caps
.max_burst
)
47 max_burst
= caps
.max_burst
;
49 max_burst
= DW_SPI_RX_BURST_LEVEL
;
51 dws
->rxburst
= min(max_burst
, def_burst
);
52 dw_writel(dws
, DW_SPI_DMARDLR
, dws
->rxburst
- 1);
54 ret
= dma_get_slave_caps(dws
->txchan
, &caps
);
55 if (!ret
&& caps
.max_burst
)
56 max_burst
= caps
.max_burst
;
58 max_burst
= DW_SPI_TX_BURST_LEVEL
;
61 * Having a Rx DMA channel serviced with higher priority than a Tx DMA
62 * channel might not be enough to provide a well balanced DMA-based
63 * SPI transfer interface. There might still be moments when the Tx DMA
64 * channel is occasionally handled faster than the Rx DMA channel.
65 * That in its turn will eventually cause the SPI Rx FIFO overflow if
66 * SPI bus speed is high enough to fill the SPI Rx FIFO in before it's
67 * cleared by the Rx DMA channel. In order to fix the problem the Tx
68 * DMA activity is intentionally slowed down by limiting the SPI Tx
69 * FIFO depth with a value twice bigger than the Tx burst length.
71 dws
->txburst
= min(max_burst
, def_burst
);
72 dw_writel(dws
, DW_SPI_DMATDLR
, dws
->txburst
);
75 static int dw_spi_dma_caps_init(struct dw_spi
*dws
)
77 struct dma_slave_caps tx
, rx
;
80 ret
= dma_get_slave_caps(dws
->txchan
, &tx
);
84 ret
= dma_get_slave_caps(dws
->rxchan
, &rx
);
88 if (!(tx
.directions
& BIT(DMA_MEM_TO_DEV
) &&
89 rx
.directions
& BIT(DMA_DEV_TO_MEM
)))
92 if (tx
.max_sg_burst
> 0 && rx
.max_sg_burst
> 0)
93 dws
->dma_sg_burst
= min(tx
.max_sg_burst
, rx
.max_sg_burst
);
94 else if (tx
.max_sg_burst
> 0)
95 dws
->dma_sg_burst
= tx
.max_sg_burst
;
96 else if (rx
.max_sg_burst
> 0)
97 dws
->dma_sg_burst
= rx
.max_sg_burst
;
99 dws
->dma_sg_burst
= 0;
102 * Assuming both channels belong to the same DMA controller hence the
103 * peripheral side address width capabilities most likely would be
106 dws
->dma_addr_widths
= tx
.dst_addr_widths
& rx
.src_addr_widths
;
111 static int dw_spi_dma_init_mfld(struct device
*dev
, struct dw_spi
*dws
)
113 struct dw_dma_slave dma_tx
= { .dst_id
= 1 }, *tx
= &dma_tx
;
114 struct dw_dma_slave dma_rx
= { .src_id
= 0 }, *rx
= &dma_rx
;
115 struct pci_dev
*dma_dev
;
120 * Get pci device for DMA controller, currently it could only
121 * be the DMA controller of Medfield
123 dma_dev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x0827, NULL
);
128 dma_cap_set(DMA_SLAVE
, mask
);
130 /* 1. Init rx channel */
131 rx
->dma_dev
= &dma_dev
->dev
;
132 dws
->rxchan
= dma_request_channel(mask
, dw_spi_dma_chan_filter
, rx
);
136 /* 2. Init tx channel */
137 tx
->dma_dev
= &dma_dev
->dev
;
138 dws
->txchan
= dma_request_channel(mask
, dw_spi_dma_chan_filter
, tx
);
142 dws
->host
->dma_rx
= dws
->rxchan
;
143 dws
->host
->dma_tx
= dws
->txchan
;
145 init_completion(&dws
->dma_completion
);
147 ret
= dw_spi_dma_caps_init(dws
);
151 dw_spi_dma_maxburst_init(dws
);
153 pci_dev_put(dma_dev
);
158 dma_release_channel(dws
->txchan
);
161 dma_release_channel(dws
->rxchan
);
164 pci_dev_put(dma_dev
);
168 static int dw_spi_dma_init_generic(struct device
*dev
, struct dw_spi
*dws
)
172 dws
->rxchan
= dma_request_chan(dev
, "rx");
173 if (IS_ERR(dws
->rxchan
)) {
174 ret
= PTR_ERR(dws
->rxchan
);
179 dws
->txchan
= dma_request_chan(dev
, "tx");
180 if (IS_ERR(dws
->txchan
)) {
181 ret
= PTR_ERR(dws
->txchan
);
186 dws
->host
->dma_rx
= dws
->rxchan
;
187 dws
->host
->dma_tx
= dws
->txchan
;
189 init_completion(&dws
->dma_completion
);
191 ret
= dw_spi_dma_caps_init(dws
);
195 dw_spi_dma_maxburst_init(dws
);
200 dma_release_channel(dws
->txchan
);
203 dma_release_channel(dws
->rxchan
);
209 static void dw_spi_dma_exit(struct dw_spi
*dws
)
212 dmaengine_terminate_sync(dws
->txchan
);
213 dma_release_channel(dws
->txchan
);
217 dmaengine_terminate_sync(dws
->rxchan
);
218 dma_release_channel(dws
->rxchan
);
222 static irqreturn_t
dw_spi_dma_transfer_handler(struct dw_spi
*dws
)
224 dw_spi_check_status(dws
, false);
226 complete(&dws
->dma_completion
);
231 static enum dma_slave_buswidth
dw_spi_dma_convert_width(u8 n_bytes
)
235 return DMA_SLAVE_BUSWIDTH_1_BYTE
;
237 return DMA_SLAVE_BUSWIDTH_2_BYTES
;
239 return DMA_SLAVE_BUSWIDTH_4_BYTES
;
241 return DMA_SLAVE_BUSWIDTH_UNDEFINED
;
245 static bool dw_spi_can_dma(struct spi_controller
*host
,
246 struct spi_device
*spi
, struct spi_transfer
*xfer
)
248 struct dw_spi
*dws
= spi_controller_get_devdata(host
);
249 enum dma_slave_buswidth dma_bus_width
;
251 if (xfer
->len
<= dws
->fifo_len
)
254 dma_bus_width
= dw_spi_dma_convert_width(dws
->n_bytes
);
256 return dws
->dma_addr_widths
& BIT(dma_bus_width
);
259 static int dw_spi_dma_wait(struct dw_spi
*dws
, unsigned int len
, u32 speed
)
261 unsigned long long ms
;
263 ms
= len
* MSEC_PER_SEC
* BITS_PER_BYTE
;
270 ms
= wait_for_completion_timeout(&dws
->dma_completion
,
271 msecs_to_jiffies(ms
));
274 dev_err(&dws
->host
->cur_msg
->spi
->dev
,
275 "DMA transaction timed out\n");
282 static inline bool dw_spi_dma_tx_busy(struct dw_spi
*dws
)
284 return !(dw_readl(dws
, DW_SPI_SR
) & DW_SPI_SR_TF_EMPT
);
287 static int dw_spi_dma_wait_tx_done(struct dw_spi
*dws
,
288 struct spi_transfer
*xfer
)
290 int retry
= DW_SPI_WAIT_RETRIES
;
291 struct spi_delay delay
;
294 nents
= dw_readl(dws
, DW_SPI_TXFLR
);
295 delay
.unit
= SPI_DELAY_UNIT_SCK
;
296 delay
.value
= nents
* dws
->n_bytes
* BITS_PER_BYTE
;
298 while (dw_spi_dma_tx_busy(dws
) && retry
--)
299 spi_delay_exec(&delay
, xfer
);
302 dev_err(&dws
->host
->dev
, "Tx hanged up\n");
310 * dws->dma_chan_busy is set before the dma transfer starts, callback for tx
311 * channel will clear a corresponding bit.
313 static void dw_spi_dma_tx_done(void *arg
)
315 struct dw_spi
*dws
= arg
;
317 clear_bit(DW_SPI_TX_BUSY
, &dws
->dma_chan_busy
);
318 if (test_bit(DW_SPI_RX_BUSY
, &dws
->dma_chan_busy
))
321 complete(&dws
->dma_completion
);
324 static int dw_spi_dma_config_tx(struct dw_spi
*dws
)
326 struct dma_slave_config txconf
;
328 memset(&txconf
, 0, sizeof(txconf
));
329 txconf
.direction
= DMA_MEM_TO_DEV
;
330 txconf
.dst_addr
= dws
->dma_addr
;
331 txconf
.dst_maxburst
= dws
->txburst
;
332 txconf
.src_addr_width
= DMA_SLAVE_BUSWIDTH_4_BYTES
;
333 txconf
.dst_addr_width
= dw_spi_dma_convert_width(dws
->n_bytes
);
334 txconf
.device_fc
= false;
336 return dmaengine_slave_config(dws
->txchan
, &txconf
);
339 static int dw_spi_dma_submit_tx(struct dw_spi
*dws
, struct scatterlist
*sgl
,
342 struct dma_async_tx_descriptor
*txdesc
;
346 txdesc
= dmaengine_prep_slave_sg(dws
->txchan
, sgl
, nents
,
348 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
352 txdesc
->callback
= dw_spi_dma_tx_done
;
353 txdesc
->callback_param
= dws
;
355 cookie
= dmaengine_submit(txdesc
);
356 ret
= dma_submit_error(cookie
);
358 dmaengine_terminate_sync(dws
->txchan
);
362 set_bit(DW_SPI_TX_BUSY
, &dws
->dma_chan_busy
);
367 static inline bool dw_spi_dma_rx_busy(struct dw_spi
*dws
)
369 return !!(dw_readl(dws
, DW_SPI_SR
) & DW_SPI_SR_RF_NOT_EMPT
);
372 static int dw_spi_dma_wait_rx_done(struct dw_spi
*dws
)
374 int retry
= DW_SPI_WAIT_RETRIES
;
375 struct spi_delay delay
;
376 unsigned long ns
, us
;
380 * It's unlikely that DMA engine is still doing the data fetching, but
381 * if it's let's give it some reasonable time. The timeout calculation
382 * is based on the synchronous APB/SSI reference clock rate, on a
383 * number of data entries left in the Rx FIFO, times a number of clock
384 * periods normally needed for a single APB read/write transaction
385 * without PREADY signal utilized (which is true for the DW APB SSI
388 nents
= dw_readl(dws
, DW_SPI_RXFLR
);
389 ns
= 4U * NSEC_PER_SEC
/ dws
->max_freq
* nents
;
390 if (ns
<= NSEC_PER_USEC
) {
391 delay
.unit
= SPI_DELAY_UNIT_NSECS
;
394 us
= DIV_ROUND_UP(ns
, NSEC_PER_USEC
);
395 delay
.unit
= SPI_DELAY_UNIT_USECS
;
396 delay
.value
= clamp_val(us
, 0, USHRT_MAX
);
399 while (dw_spi_dma_rx_busy(dws
) && retry
--)
400 spi_delay_exec(&delay
, NULL
);
403 dev_err(&dws
->host
->dev
, "Rx hanged up\n");
411 * dws->dma_chan_busy is set before the dma transfer starts, callback for rx
412 * channel will clear a corresponding bit.
414 static void dw_spi_dma_rx_done(void *arg
)
416 struct dw_spi
*dws
= arg
;
418 clear_bit(DW_SPI_RX_BUSY
, &dws
->dma_chan_busy
);
419 if (test_bit(DW_SPI_TX_BUSY
, &dws
->dma_chan_busy
))
422 complete(&dws
->dma_completion
);
425 static int dw_spi_dma_config_rx(struct dw_spi
*dws
)
427 struct dma_slave_config rxconf
;
429 memset(&rxconf
, 0, sizeof(rxconf
));
430 rxconf
.direction
= DMA_DEV_TO_MEM
;
431 rxconf
.src_addr
= dws
->dma_addr
;
432 rxconf
.src_maxburst
= dws
->rxburst
;
433 rxconf
.dst_addr_width
= DMA_SLAVE_BUSWIDTH_4_BYTES
;
434 rxconf
.src_addr_width
= dw_spi_dma_convert_width(dws
->n_bytes
);
435 rxconf
.device_fc
= false;
437 return dmaengine_slave_config(dws
->rxchan
, &rxconf
);
440 static int dw_spi_dma_submit_rx(struct dw_spi
*dws
, struct scatterlist
*sgl
,
443 struct dma_async_tx_descriptor
*rxdesc
;
447 rxdesc
= dmaengine_prep_slave_sg(dws
->rxchan
, sgl
, nents
,
449 DMA_PREP_INTERRUPT
| DMA_CTRL_ACK
);
453 rxdesc
->callback
= dw_spi_dma_rx_done
;
454 rxdesc
->callback_param
= dws
;
456 cookie
= dmaengine_submit(rxdesc
);
457 ret
= dma_submit_error(cookie
);
459 dmaengine_terminate_sync(dws
->rxchan
);
463 set_bit(DW_SPI_RX_BUSY
, &dws
->dma_chan_busy
);
468 static int dw_spi_dma_setup(struct dw_spi
*dws
, struct spi_transfer
*xfer
)
476 /* Setup DMA channels */
477 ret
= dw_spi_dma_config_tx(dws
);
482 ret
= dw_spi_dma_config_rx(dws
);
487 /* Set the DMA handshaking interface */
488 dma_ctrl
= DW_SPI_DMACR_TDMAE
;
490 dma_ctrl
|= DW_SPI_DMACR_RDMAE
;
491 dw_writel(dws
, DW_SPI_DMACR
, dma_ctrl
);
493 /* Set the interrupt mask */
494 imr
= DW_SPI_INT_TXOI
;
496 imr
|= DW_SPI_INT_RXUI
| DW_SPI_INT_RXOI
;
497 dw_spi_umask_intr(dws
, imr
);
499 reinit_completion(&dws
->dma_completion
);
501 dws
->transfer_handler
= dw_spi_dma_transfer_handler
;
506 static int dw_spi_dma_transfer_all(struct dw_spi
*dws
,
507 struct spi_transfer
*xfer
)
511 /* Submit the DMA Tx transfer */
512 ret
= dw_spi_dma_submit_tx(dws
, xfer
->tx_sg
.sgl
, xfer
->tx_sg
.nents
);
516 /* Submit the DMA Rx transfer if required */
518 ret
= dw_spi_dma_submit_rx(dws
, xfer
->rx_sg
.sgl
,
523 /* rx must be started before tx due to spi instinct */
524 dma_async_issue_pending(dws
->rxchan
);
527 dma_async_issue_pending(dws
->txchan
);
529 ret
= dw_spi_dma_wait(dws
, xfer
->len
, xfer
->effective_speed_hz
);
532 dw_writel(dws
, DW_SPI_DMACR
, 0);
538 * In case if at least one of the requested DMA channels doesn't support the
539 * hardware accelerated SG list entries traverse, the DMA driver will most
540 * likely work that around by performing the IRQ-based SG list entries
541 * resubmission. That might and will cause a problem if the DMA Tx channel is
542 * recharged and re-executed before the Rx DMA channel. Due to
543 * non-deterministic IRQ-handler execution latency the DMA Tx channel will
544 * start pushing data to the SPI bus before the Rx DMA channel is even
545 * reinitialized with the next inbound SG list entry. By doing so the DMA Tx
546 * channel will implicitly start filling the DW APB SSI Rx FIFO up, which while
547 * the DMA Rx channel being recharged and re-executed will eventually be
550 * In order to solve the problem we have to feed the DMA engine with SG list
551 * entries one-by-one. It shall keep the DW APB SSI Tx and Rx FIFOs
552 * synchronized and prevent the Rx FIFO overflow. Since in general the tx_sg
553 * and rx_sg lists may have different number of entries of different lengths
554 * (though total length should match) let's virtually split the SG-lists to the
555 * set of DMA transfers, which length is a minimum of the ordered SG-entries
556 * lengths. An ASCII-sketch of the implemented algo is following:
559 * tx_sg list: |___|____|__|
560 * rx_sg list: |_|____|____|
561 * DMA transfers: |_|_|__|_|__|
563 * Note in order to have this workaround solving the denoted problem the DMA
564 * engine driver should properly initialize the max_sg_burst capability and set
565 * the DMA device max segment size parameter with maximum data block size the
566 * DMA engine supports.
569 static int dw_spi_dma_transfer_one(struct dw_spi
*dws
,
570 struct spi_transfer
*xfer
)
572 struct scatterlist
*tx_sg
= NULL
, *rx_sg
= NULL
, tx_tmp
, rx_tmp
;
573 unsigned int tx_len
= 0, rx_len
= 0;
574 unsigned int base
, len
;
577 sg_init_table(&tx_tmp
, 1);
578 sg_init_table(&rx_tmp
, 1);
580 for (base
= 0; base
< xfer
->len
; base
+= len
) {
581 /* Fetch next Tx DMA data chunk */
583 tx_sg
= !tx_sg
? &xfer
->tx_sg
.sgl
[0] : sg_next(tx_sg
);
584 sg_dma_address(&tx_tmp
) = sg_dma_address(tx_sg
);
585 tx_len
= sg_dma_len(tx_sg
);
588 /* Fetch next Rx DMA data chunk */
590 rx_sg
= !rx_sg
? &xfer
->rx_sg
.sgl
[0] : sg_next(rx_sg
);
591 sg_dma_address(&rx_tmp
) = sg_dma_address(rx_sg
);
592 rx_len
= sg_dma_len(rx_sg
);
595 len
= min(tx_len
, rx_len
);
597 sg_dma_len(&tx_tmp
) = len
;
598 sg_dma_len(&rx_tmp
) = len
;
600 /* Submit DMA Tx transfer */
601 ret
= dw_spi_dma_submit_tx(dws
, &tx_tmp
, 1);
605 /* Submit DMA Rx transfer */
606 ret
= dw_spi_dma_submit_rx(dws
, &rx_tmp
, 1);
610 /* Rx must be started before Tx due to SPI instinct */
611 dma_async_issue_pending(dws
->rxchan
);
613 dma_async_issue_pending(dws
->txchan
);
616 * Here we only need to wait for the DMA transfer to be
617 * finished since SPI controller is kept enabled during the
618 * procedure this loop implements and there is no risk to lose
619 * data left in the Tx/Rx FIFOs.
621 ret
= dw_spi_dma_wait(dws
, len
, xfer
->effective_speed_hz
);
625 reinit_completion(&dws
->dma_completion
);
627 sg_dma_address(&tx_tmp
) += len
;
628 sg_dma_address(&rx_tmp
) += len
;
633 dw_writel(dws
, DW_SPI_DMACR
, 0);
638 static int dw_spi_dma_transfer(struct dw_spi
*dws
, struct spi_transfer
*xfer
)
643 nents
= max(xfer
->tx_sg
.nents
, xfer
->rx_sg
.nents
);
646 * Execute normal DMA-based transfer (which submits the Rx and Tx SG
647 * lists directly to the DMA engine at once) if either full hardware
648 * accelerated SG list traverse is supported by both channels, or the
649 * Tx-only SPI transfer is requested, or the DMA engine is capable to
650 * handle both SG lists on hardware accelerated basis.
652 if (!dws
->dma_sg_burst
|| !xfer
->rx_buf
|| nents
<= dws
->dma_sg_burst
)
653 ret
= dw_spi_dma_transfer_all(dws
, xfer
);
655 ret
= dw_spi_dma_transfer_one(dws
, xfer
);
659 if (dws
->host
->cur_msg
->status
== -EINPROGRESS
) {
660 ret
= dw_spi_dma_wait_tx_done(dws
, xfer
);
665 if (xfer
->rx_buf
&& dws
->host
->cur_msg
->status
== -EINPROGRESS
)
666 ret
= dw_spi_dma_wait_rx_done(dws
);
671 static void dw_spi_dma_stop(struct dw_spi
*dws
)
673 if (test_bit(DW_SPI_TX_BUSY
, &dws
->dma_chan_busy
)) {
674 dmaengine_terminate_sync(dws
->txchan
);
675 clear_bit(DW_SPI_TX_BUSY
, &dws
->dma_chan_busy
);
677 if (test_bit(DW_SPI_RX_BUSY
, &dws
->dma_chan_busy
)) {
678 dmaengine_terminate_sync(dws
->rxchan
);
679 clear_bit(DW_SPI_RX_BUSY
, &dws
->dma_chan_busy
);
683 static const struct dw_spi_dma_ops dw_spi_dma_mfld_ops
= {
684 .dma_init
= dw_spi_dma_init_mfld
,
685 .dma_exit
= dw_spi_dma_exit
,
686 .dma_setup
= dw_spi_dma_setup
,
687 .can_dma
= dw_spi_can_dma
,
688 .dma_transfer
= dw_spi_dma_transfer
,
689 .dma_stop
= dw_spi_dma_stop
,
692 void dw_spi_dma_setup_mfld(struct dw_spi
*dws
)
694 dws
->dma_ops
= &dw_spi_dma_mfld_ops
;
696 EXPORT_SYMBOL_NS_GPL(dw_spi_dma_setup_mfld
, "SPI_DW_CORE");
698 static const struct dw_spi_dma_ops dw_spi_dma_generic_ops
= {
699 .dma_init
= dw_spi_dma_init_generic
,
700 .dma_exit
= dw_spi_dma_exit
,
701 .dma_setup
= dw_spi_dma_setup
,
702 .can_dma
= dw_spi_can_dma
,
703 .dma_transfer
= dw_spi_dma_transfer
,
704 .dma_stop
= dw_spi_dma_stop
,
707 void dw_spi_dma_setup_generic(struct dw_spi
*dws
)
709 dws
->dma_ops
= &dw_spi_dma_generic_ops
;
711 EXPORT_SYMBOL_NS_GPL(dw_spi_dma_setup_generic
, "SPI_DW_CORE");