2 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
6 * License terms: GNU General Public License (GPL) version 2
9 #include <linux/dma-mapping.h>
10 #include <linux/kernel.h>
11 #include <linux/slab.h>
12 #include <linux/export.h>
13 #include <linux/dmaengine.h>
14 #include <linux/platform_device.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/log2.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/err.h>
22 #include <linux/of_dma.h>
23 #include <linux/amba/bus.h>
24 #include <linux/regulator/consumer.h>
25 #include <linux/platform_data/dma-ste-dma40.h>
27 #include "dmaengine.h"
28 #include "ste_dma40_ll.h"
30 #define D40_NAME "dma40"
32 #define D40_PHY_CHAN -1
34 /* For masking out/in 2 bit channel positions */
35 #define D40_CHAN_POS(chan) (2 * (chan / 2))
36 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
38 /* Maximum iterations taken before giving up suspending a channel */
39 #define D40_SUSPEND_MAX_IT 500
42 #define DMA40_AUTOSUSPEND_DELAY 100
44 /* Hardware requirement on LCLA alignment */
45 #define LCLA_ALIGNMENT 0x40000
47 /* Max number of links per event group */
48 #define D40_LCLA_LINK_PER_EVENT_GRP 128
49 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
51 /* Max number of logical channels per physical channel */
52 #define D40_MAX_LOG_CHAN_PER_PHY 32
54 /* Attempts before giving up to trying to get pages that are aligned */
55 #define MAX_LCLA_ALLOC_ATTEMPTS 256
57 /* Bit markings for allocation map */
58 #define D40_ALLOC_FREE BIT(31)
59 #define D40_ALLOC_PHY BIT(30)
60 #define D40_ALLOC_LOG_FREE 0
62 #define D40_MEMCPY_MAX_CHANS 8
64 /* Reserved event lines for memcpy only. */
65 #define DB8500_DMA_MEMCPY_EV_0 51
66 #define DB8500_DMA_MEMCPY_EV_1 56
67 #define DB8500_DMA_MEMCPY_EV_2 57
68 #define DB8500_DMA_MEMCPY_EV_3 58
69 #define DB8500_DMA_MEMCPY_EV_4 59
70 #define DB8500_DMA_MEMCPY_EV_5 60
72 static int dma40_memcpy_channels
[] = {
73 DB8500_DMA_MEMCPY_EV_0
,
74 DB8500_DMA_MEMCPY_EV_1
,
75 DB8500_DMA_MEMCPY_EV_2
,
76 DB8500_DMA_MEMCPY_EV_3
,
77 DB8500_DMA_MEMCPY_EV_4
,
78 DB8500_DMA_MEMCPY_EV_5
,
81 /* Default configuration for physcial memcpy */
82 static struct stedma40_chan_cfg dma40_memcpy_conf_phy
= {
83 .mode
= STEDMA40_MODE_PHYSICAL
,
84 .dir
= DMA_MEM_TO_MEM
,
86 .src_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
87 .src_info
.psize
= STEDMA40_PSIZE_PHY_1
,
88 .src_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
90 .dst_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
91 .dst_info
.psize
= STEDMA40_PSIZE_PHY_1
,
92 .dst_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
95 /* Default configuration for logical memcpy */
96 static struct stedma40_chan_cfg dma40_memcpy_conf_log
= {
97 .mode
= STEDMA40_MODE_LOGICAL
,
98 .dir
= DMA_MEM_TO_MEM
,
100 .src_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
101 .src_info
.psize
= STEDMA40_PSIZE_LOG_1
,
102 .src_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
104 .dst_info
.data_width
= DMA_SLAVE_BUSWIDTH_1_BYTE
,
105 .dst_info
.psize
= STEDMA40_PSIZE_LOG_1
,
106 .dst_info
.flow_ctrl
= STEDMA40_NO_FLOW_CTRL
,
110 * enum 40_command - The different commands and/or statuses.
112 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
113 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
114 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
115 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
120 D40_DMA_SUSPEND_REQ
= 2,
121 D40_DMA_SUSPENDED
= 3
125 * enum d40_events - The different Event Enables for the event lines.
127 * @D40_DEACTIVATE_EVENTLINE: De-activate Event line, stopping the logical chan.
128 * @D40_ACTIVATE_EVENTLINE: Activate the Event line, to start a logical chan.
129 * @D40_SUSPEND_REQ_EVENTLINE: Requesting for suspending a event line.
130 * @D40_ROUND_EVENTLINE: Status check for event line.
134 D40_DEACTIVATE_EVENTLINE
= 0,
135 D40_ACTIVATE_EVENTLINE
= 1,
136 D40_SUSPEND_REQ_EVENTLINE
= 2,
137 D40_ROUND_EVENTLINE
= 3
141 * These are the registers that has to be saved and later restored
142 * when the DMA hw is powered off.
143 * TODO: Add save/restore of D40_DREG_GCC on dma40 v3 or later, if that works.
145 static u32 d40_backup_regs
[] = {
154 #define BACKUP_REGS_SZ ARRAY_SIZE(d40_backup_regs)
157 * since 9540 and 8540 has the same HW revision
158 * use v4a for 9540 or ealier
159 * use v4b for 8540 or later
161 * DB8500ed has revision 0
162 * DB8500v1 has revision 2
163 * DB8500v2 has revision 3
164 * AP9540v1 has revision 4
165 * DB8540v1 has revision 4
166 * TODO: Check if all these registers have to be saved/restored on dma40 v4a
168 static u32 d40_backup_regs_v4a
[] = {
187 #define BACKUP_REGS_SZ_V4A ARRAY_SIZE(d40_backup_regs_v4a)
189 static u32 d40_backup_regs_v4b
[] = {
212 #define BACKUP_REGS_SZ_V4B ARRAY_SIZE(d40_backup_regs_v4b)
214 static u32 d40_backup_regs_chan
[] = {
225 #define BACKUP_REGS_SZ_MAX ((BACKUP_REGS_SZ_V4A > BACKUP_REGS_SZ_V4B) ? \
226 BACKUP_REGS_SZ_V4A : BACKUP_REGS_SZ_V4B)
229 * struct d40_interrupt_lookup - lookup table for interrupt handler
231 * @src: Interrupt mask register.
232 * @clr: Interrupt clear register.
233 * @is_error: true if this is an error interrupt.
234 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
235 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
237 struct d40_interrupt_lookup
{
245 static struct d40_interrupt_lookup il_v4a
[] = {
246 {D40_DREG_LCTIS0
, D40_DREG_LCICR0
, false, 0},
247 {D40_DREG_LCTIS1
, D40_DREG_LCICR1
, false, 32},
248 {D40_DREG_LCTIS2
, D40_DREG_LCICR2
, false, 64},
249 {D40_DREG_LCTIS3
, D40_DREG_LCICR3
, false, 96},
250 {D40_DREG_LCEIS0
, D40_DREG_LCICR0
, true, 0},
251 {D40_DREG_LCEIS1
, D40_DREG_LCICR1
, true, 32},
252 {D40_DREG_LCEIS2
, D40_DREG_LCICR2
, true, 64},
253 {D40_DREG_LCEIS3
, D40_DREG_LCICR3
, true, 96},
254 {D40_DREG_PCTIS
, D40_DREG_PCICR
, false, D40_PHY_CHAN
},
255 {D40_DREG_PCEIS
, D40_DREG_PCICR
, true, D40_PHY_CHAN
},
258 static struct d40_interrupt_lookup il_v4b
[] = {
259 {D40_DREG_CLCTIS1
, D40_DREG_CLCICR1
, false, 0},
260 {D40_DREG_CLCTIS2
, D40_DREG_CLCICR2
, false, 32},
261 {D40_DREG_CLCTIS3
, D40_DREG_CLCICR3
, false, 64},
262 {D40_DREG_CLCTIS4
, D40_DREG_CLCICR4
, false, 96},
263 {D40_DREG_CLCTIS5
, D40_DREG_CLCICR5
, false, 128},
264 {D40_DREG_CLCEIS1
, D40_DREG_CLCICR1
, true, 0},
265 {D40_DREG_CLCEIS2
, D40_DREG_CLCICR2
, true, 32},
266 {D40_DREG_CLCEIS3
, D40_DREG_CLCICR3
, true, 64},
267 {D40_DREG_CLCEIS4
, D40_DREG_CLCICR4
, true, 96},
268 {D40_DREG_CLCEIS5
, D40_DREG_CLCICR5
, true, 128},
269 {D40_DREG_CPCTIS
, D40_DREG_CPCICR
, false, D40_PHY_CHAN
},
270 {D40_DREG_CPCEIS
, D40_DREG_CPCICR
, true, D40_PHY_CHAN
},
274 * struct d40_reg_val - simple lookup struct
276 * @reg: The register.
277 * @val: The value that belongs to the register in reg.
284 static __initdata
struct d40_reg_val dma_init_reg_v4a
[] = {
285 /* Clock every part of the DMA block from start */
286 { .reg
= D40_DREG_GCC
, .val
= D40_DREG_GCC_ENABLE_ALL
},
288 /* Interrupts on all logical channels */
289 { .reg
= D40_DREG_LCMIS0
, .val
= 0xFFFFFFFF},
290 { .reg
= D40_DREG_LCMIS1
, .val
= 0xFFFFFFFF},
291 { .reg
= D40_DREG_LCMIS2
, .val
= 0xFFFFFFFF},
292 { .reg
= D40_DREG_LCMIS3
, .val
= 0xFFFFFFFF},
293 { .reg
= D40_DREG_LCICR0
, .val
= 0xFFFFFFFF},
294 { .reg
= D40_DREG_LCICR1
, .val
= 0xFFFFFFFF},
295 { .reg
= D40_DREG_LCICR2
, .val
= 0xFFFFFFFF},
296 { .reg
= D40_DREG_LCICR3
, .val
= 0xFFFFFFFF},
297 { .reg
= D40_DREG_LCTIS0
, .val
= 0xFFFFFFFF},
298 { .reg
= D40_DREG_LCTIS1
, .val
= 0xFFFFFFFF},
299 { .reg
= D40_DREG_LCTIS2
, .val
= 0xFFFFFFFF},
300 { .reg
= D40_DREG_LCTIS3
, .val
= 0xFFFFFFFF}
302 static __initdata
struct d40_reg_val dma_init_reg_v4b
[] = {
303 /* Clock every part of the DMA block from start */
304 { .reg
= D40_DREG_GCC
, .val
= D40_DREG_GCC_ENABLE_ALL
},
306 /* Interrupts on all logical channels */
307 { .reg
= D40_DREG_CLCMIS1
, .val
= 0xFFFFFFFF},
308 { .reg
= D40_DREG_CLCMIS2
, .val
= 0xFFFFFFFF},
309 { .reg
= D40_DREG_CLCMIS3
, .val
= 0xFFFFFFFF},
310 { .reg
= D40_DREG_CLCMIS4
, .val
= 0xFFFFFFFF},
311 { .reg
= D40_DREG_CLCMIS5
, .val
= 0xFFFFFFFF},
312 { .reg
= D40_DREG_CLCICR1
, .val
= 0xFFFFFFFF},
313 { .reg
= D40_DREG_CLCICR2
, .val
= 0xFFFFFFFF},
314 { .reg
= D40_DREG_CLCICR3
, .val
= 0xFFFFFFFF},
315 { .reg
= D40_DREG_CLCICR4
, .val
= 0xFFFFFFFF},
316 { .reg
= D40_DREG_CLCICR5
, .val
= 0xFFFFFFFF},
317 { .reg
= D40_DREG_CLCTIS1
, .val
= 0xFFFFFFFF},
318 { .reg
= D40_DREG_CLCTIS2
, .val
= 0xFFFFFFFF},
319 { .reg
= D40_DREG_CLCTIS3
, .val
= 0xFFFFFFFF},
320 { .reg
= D40_DREG_CLCTIS4
, .val
= 0xFFFFFFFF},
321 { .reg
= D40_DREG_CLCTIS5
, .val
= 0xFFFFFFFF}
325 * struct d40_lli_pool - Structure for keeping LLIs in memory
327 * @base: Pointer to memory area when the pre_alloc_lli's are not large
328 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
329 * pre_alloc_lli is used.
330 * @dma_addr: DMA address, if mapped
331 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
332 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
333 * one buffer to one buffer.
335 struct d40_lli_pool
{
339 /* Space for dst and src, plus an extra for padding */
340 u8 pre_alloc_lli
[3 * sizeof(struct d40_phy_lli
)];
344 * struct d40_desc - A descriptor is one DMA job.
346 * @lli_phy: LLI settings for physical channel. Both src and dst=
347 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
348 * lli_len equals one.
349 * @lli_log: Same as above but for logical channels.
350 * @lli_pool: The pool with two entries pre-allocated.
351 * @lli_len: Number of llis of current descriptor.
352 * @lli_current: Number of transferred llis.
353 * @lcla_alloc: Number of LCLA entries allocated.
354 * @txd: DMA engine struct. Used for among other things for communication
357 * @is_in_client_list: true if the client owns this descriptor.
358 * @cyclic: true if this is a cyclic job
360 * This descriptor is used for both logical and physical transfers.
364 struct d40_phy_lli_bidir lli_phy
;
366 struct d40_log_lli_bidir lli_log
;
368 struct d40_lli_pool lli_pool
;
373 struct dma_async_tx_descriptor txd
;
374 struct list_head node
;
376 bool is_in_client_list
;
381 * struct d40_lcla_pool - LCLA pool settings and data.
383 * @base: The virtual address of LCLA. 18 bit aligned.
384 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
385 * This pointer is only there for clean-up on error.
386 * @pages: The number of pages needed for all physical channels.
387 * Only used later for clean-up on error
388 * @lock: Lock to protect the content in this struct.
389 * @alloc_map: big map over which LCLA entry is own by which job.
391 struct d40_lcla_pool
{
394 void *base_unaligned
;
397 struct d40_desc
**alloc_map
;
401 * struct d40_phy_res - struct for handling eventlines mapped to physical
404 * @lock: A lock protection this entity.
405 * @reserved: True if used by secure world or otherwise.
406 * @num: The physical channel number of this entity.
407 * @allocated_src: Bit mapped to show which src event line's are mapped to
408 * this physical channel. Can also be free or physically allocated.
409 * @allocated_dst: Same as for src but is dst.
410 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
412 * @use_soft_lli: To mark if the linked lists of channel are managed by SW.
426 * struct d40_chan - Struct that describes a channel.
428 * @lock: A spinlock to protect this struct.
429 * @log_num: The logical number, if any of this channel.
430 * @pending_tx: The number of pending transfers. Used between interrupt handler
432 * @busy: Set to true when transfer is ongoing on this channel.
433 * @phy_chan: Pointer to physical channel which this instance runs on. If this
434 * point is NULL, then the channel is not allocated.
435 * @chan: DMA engine handle.
436 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
437 * transfer and call client callback.
438 * @client: Cliented owned descriptor list.
439 * @pending_queue: Submitted jobs, to be issued by issue_pending()
440 * @active: Active descriptor.
441 * @done: Completed jobs
442 * @queue: Queued jobs.
443 * @prepare_queue: Prepared jobs.
444 * @dma_cfg: The client configuration of this dma channel.
445 * @configured: whether the dma_cfg configuration is valid
446 * @base: Pointer to the device instance struct.
447 * @src_def_cfg: Default cfg register setting for src.
448 * @dst_def_cfg: Default cfg register setting for dst.
449 * @log_def: Default logical channel settings.
450 * @lcpa: Pointer to dst and src lcpa settings.
451 * @runtime_addr: runtime configured address.
452 * @runtime_direction: runtime configured direction.
454 * This struct can either "be" a logical or a physical channel.
461 struct d40_phy_res
*phy_chan
;
462 struct dma_chan chan
;
463 struct tasklet_struct tasklet
;
464 struct list_head client
;
465 struct list_head pending_queue
;
466 struct list_head active
;
467 struct list_head done
;
468 struct list_head queue
;
469 struct list_head prepare_queue
;
470 struct stedma40_chan_cfg dma_cfg
;
472 struct d40_base
*base
;
473 /* Default register configurations */
476 struct d40_def_lcsp log_def
;
477 struct d40_log_lli_full
*lcpa
;
478 /* Runtime reconfiguration */
479 dma_addr_t runtime_addr
;
480 enum dma_transfer_direction runtime_direction
;
484 * struct d40_gen_dmac - generic values to represent u8500/u8540 DMA
487 * @backup: the pointer to the registers address array for backup
488 * @backup_size: the size of the registers address array for backup
489 * @realtime_en: the realtime enable register
490 * @realtime_clear: the realtime clear register
491 * @high_prio_en: the high priority enable register
492 * @high_prio_clear: the high priority clear register
493 * @interrupt_en: the interrupt enable register
494 * @interrupt_clear: the interrupt clear register
495 * @il: the pointer to struct d40_interrupt_lookup
496 * @il_size: the size of d40_interrupt_lookup array
497 * @init_reg: the pointer to the struct d40_reg_val
498 * @init_reg_size: the size of d40_reg_val array
500 struct d40_gen_dmac
{
509 struct d40_interrupt_lookup
*il
;
511 struct d40_reg_val
*init_reg
;
516 * struct d40_base - The big global struct, one for each probe'd instance.
518 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
519 * @execmd_lock: Lock for execute command usage since several channels share
520 * the same physical register.
521 * @dev: The device structure.
522 * @virtbase: The virtual base address of the DMA's register.
523 * @rev: silicon revision detected.
524 * @clk: Pointer to the DMA clock structure.
525 * @phy_start: Physical memory start of the DMA registers.
526 * @phy_size: Size of the DMA register map.
527 * @irq: The IRQ number.
528 * @num_memcpy_chans: The number of channels used for memcpy (mem-to-mem
530 * @num_phy_chans: The number of physical channels. Read from HW. This
531 * is the number of available channels for this driver, not counting "Secure
532 * mode" allocated physical channels.
533 * @num_log_chans: The number of logical channels. Calculated from
535 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
536 * @dma_slave: dma_device channels that can do only do slave transfers.
537 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
538 * @phy_chans: Room for all possible physical channels in system.
539 * @log_chans: Room for all possible logical channels in system.
540 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
541 * to log_chans entries.
542 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
543 * to phy_chans entries.
544 * @plat_data: Pointer to provided platform_data which is the driver
546 * @lcpa_regulator: Pointer to hold the regulator for the esram bank for lcla.
547 * @phy_res: Vector containing all physical channels.
548 * @lcla_pool: lcla pool settings and data.
549 * @lcpa_base: The virtual mapped address of LCPA.
550 * @phy_lcpa: The physical address of the LCPA.
551 * @lcpa_size: The size of the LCPA area.
552 * @desc_slab: cache for descriptors.
553 * @reg_val_backup: Here the values of some hardware registers are stored
554 * before the DMA is powered off. They are restored when the power is back on.
555 * @reg_val_backup_v4: Backup of registers that only exits on dma40 v3 and
557 * @reg_val_backup_chan: Backup data for standard channel parameter registers.
558 * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off.
559 * @gen_dmac: the struct for generic registers values to represent u8500/8540
563 spinlock_t interrupt_lock
;
564 spinlock_t execmd_lock
;
566 void __iomem
*virtbase
;
569 phys_addr_t phy_start
;
570 resource_size_t phy_size
;
572 int num_memcpy_chans
;
575 struct device_dma_parameters dma_parms
;
576 struct dma_device dma_both
;
577 struct dma_device dma_slave
;
578 struct dma_device dma_memcpy
;
579 struct d40_chan
*phy_chans
;
580 struct d40_chan
*log_chans
;
581 struct d40_chan
**lookup_log_chans
;
582 struct d40_chan
**lookup_phy_chans
;
583 struct stedma40_platform_data
*plat_data
;
584 struct regulator
*lcpa_regulator
;
585 /* Physical half channels */
586 struct d40_phy_res
*phy_res
;
587 struct d40_lcla_pool lcla_pool
;
590 resource_size_t lcpa_size
;
591 struct kmem_cache
*desc_slab
;
592 u32 reg_val_backup
[BACKUP_REGS_SZ
];
593 u32 reg_val_backup_v4
[BACKUP_REGS_SZ_MAX
];
594 u32
*reg_val_backup_chan
;
595 u16 gcc_pwr_off_mask
;
596 struct d40_gen_dmac gen_dmac
;
599 static struct device
*chan2dev(struct d40_chan
*d40c
)
601 return &d40c
->chan
.dev
->device
;
604 static bool chan_is_physical(struct d40_chan
*chan
)
606 return chan
->log_num
== D40_PHY_CHAN
;
609 static bool chan_is_logical(struct d40_chan
*chan
)
611 return !chan_is_physical(chan
);
614 static void __iomem
*chan_base(struct d40_chan
*chan
)
616 return chan
->base
->virtbase
+ D40_DREG_PCBASE
+
617 chan
->phy_chan
->num
* D40_DREG_PCDELTA
;
620 #define d40_err(dev, format, arg...) \
621 dev_err(dev, "[%s] " format, __func__, ## arg)
623 #define chan_err(d40c, format, arg...) \
624 d40_err(chan2dev(d40c), format, ## arg)
626 static int d40_pool_lli_alloc(struct d40_chan
*d40c
, struct d40_desc
*d40d
,
629 bool is_log
= chan_is_logical(d40c
);
634 align
= sizeof(struct d40_log_lli
);
636 align
= sizeof(struct d40_phy_lli
);
639 base
= d40d
->lli_pool
.pre_alloc_lli
;
640 d40d
->lli_pool
.size
= sizeof(d40d
->lli_pool
.pre_alloc_lli
);
641 d40d
->lli_pool
.base
= NULL
;
643 d40d
->lli_pool
.size
= lli_len
* 2 * align
;
645 base
= kmalloc(d40d
->lli_pool
.size
+ align
, GFP_NOWAIT
);
646 d40d
->lli_pool
.base
= base
;
648 if (d40d
->lli_pool
.base
== NULL
)
653 d40d
->lli_log
.src
= PTR_ALIGN(base
, align
);
654 d40d
->lli_log
.dst
= d40d
->lli_log
.src
+ lli_len
;
656 d40d
->lli_pool
.dma_addr
= 0;
658 d40d
->lli_phy
.src
= PTR_ALIGN(base
, align
);
659 d40d
->lli_phy
.dst
= d40d
->lli_phy
.src
+ lli_len
;
661 d40d
->lli_pool
.dma_addr
= dma_map_single(d40c
->base
->dev
,
666 if (dma_mapping_error(d40c
->base
->dev
,
667 d40d
->lli_pool
.dma_addr
)) {
668 kfree(d40d
->lli_pool
.base
);
669 d40d
->lli_pool
.base
= NULL
;
670 d40d
->lli_pool
.dma_addr
= 0;
678 static void d40_pool_lli_free(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
680 if (d40d
->lli_pool
.dma_addr
)
681 dma_unmap_single(d40c
->base
->dev
, d40d
->lli_pool
.dma_addr
,
682 d40d
->lli_pool
.size
, DMA_TO_DEVICE
);
684 kfree(d40d
->lli_pool
.base
);
685 d40d
->lli_pool
.base
= NULL
;
686 d40d
->lli_pool
.size
= 0;
687 d40d
->lli_log
.src
= NULL
;
688 d40d
->lli_log
.dst
= NULL
;
689 d40d
->lli_phy
.src
= NULL
;
690 d40d
->lli_phy
.dst
= NULL
;
693 static int d40_lcla_alloc_one(struct d40_chan
*d40c
,
694 struct d40_desc
*d40d
)
700 spin_lock_irqsave(&d40c
->base
->lcla_pool
.lock
, flags
);
703 * Allocate both src and dst at the same time, therefore the half
704 * start on 1 since 0 can't be used since zero is used as end marker.
706 for (i
= 1 ; i
< D40_LCLA_LINK_PER_EVENT_GRP
/ 2; i
++) {
707 int idx
= d40c
->phy_chan
->num
* D40_LCLA_LINK_PER_EVENT_GRP
+ i
;
709 if (!d40c
->base
->lcla_pool
.alloc_map
[idx
]) {
710 d40c
->base
->lcla_pool
.alloc_map
[idx
] = d40d
;
717 spin_unlock_irqrestore(&d40c
->base
->lcla_pool
.lock
, flags
);
722 static int d40_lcla_free_all(struct d40_chan
*d40c
,
723 struct d40_desc
*d40d
)
729 if (chan_is_physical(d40c
))
732 spin_lock_irqsave(&d40c
->base
->lcla_pool
.lock
, flags
);
734 for (i
= 1 ; i
< D40_LCLA_LINK_PER_EVENT_GRP
/ 2; i
++) {
735 int idx
= d40c
->phy_chan
->num
* D40_LCLA_LINK_PER_EVENT_GRP
+ i
;
737 if (d40c
->base
->lcla_pool
.alloc_map
[idx
] == d40d
) {
738 d40c
->base
->lcla_pool
.alloc_map
[idx
] = NULL
;
740 if (d40d
->lcla_alloc
== 0) {
747 spin_unlock_irqrestore(&d40c
->base
->lcla_pool
.lock
, flags
);
753 static void d40_desc_remove(struct d40_desc
*d40d
)
755 list_del(&d40d
->node
);
758 static struct d40_desc
*d40_desc_get(struct d40_chan
*d40c
)
760 struct d40_desc
*desc
= NULL
;
762 if (!list_empty(&d40c
->client
)) {
766 list_for_each_entry_safe(d
, _d
, &d40c
->client
, node
) {
767 if (async_tx_test_ack(&d
->txd
)) {
770 memset(desc
, 0, sizeof(*desc
));
777 desc
= kmem_cache_zalloc(d40c
->base
->desc_slab
, GFP_NOWAIT
);
780 INIT_LIST_HEAD(&desc
->node
);
785 static void d40_desc_free(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
788 d40_pool_lli_free(d40c
, d40d
);
789 d40_lcla_free_all(d40c
, d40d
);
790 kmem_cache_free(d40c
->base
->desc_slab
, d40d
);
793 static void d40_desc_submit(struct d40_chan
*d40c
, struct d40_desc
*desc
)
795 list_add_tail(&desc
->node
, &d40c
->active
);
798 static void d40_phy_lli_load(struct d40_chan
*chan
, struct d40_desc
*desc
)
800 struct d40_phy_lli
*lli_dst
= desc
->lli_phy
.dst
;
801 struct d40_phy_lli
*lli_src
= desc
->lli_phy
.src
;
802 void __iomem
*base
= chan_base(chan
);
804 writel(lli_src
->reg_cfg
, base
+ D40_CHAN_REG_SSCFG
);
805 writel(lli_src
->reg_elt
, base
+ D40_CHAN_REG_SSELT
);
806 writel(lli_src
->reg_ptr
, base
+ D40_CHAN_REG_SSPTR
);
807 writel(lli_src
->reg_lnk
, base
+ D40_CHAN_REG_SSLNK
);
809 writel(lli_dst
->reg_cfg
, base
+ D40_CHAN_REG_SDCFG
);
810 writel(lli_dst
->reg_elt
, base
+ D40_CHAN_REG_SDELT
);
811 writel(lli_dst
->reg_ptr
, base
+ D40_CHAN_REG_SDPTR
);
812 writel(lli_dst
->reg_lnk
, base
+ D40_CHAN_REG_SDLNK
);
815 static void d40_desc_done(struct d40_chan
*d40c
, struct d40_desc
*desc
)
817 list_add_tail(&desc
->node
, &d40c
->done
);
820 static void d40_log_lli_to_lcxa(struct d40_chan
*chan
, struct d40_desc
*desc
)
822 struct d40_lcla_pool
*pool
= &chan
->base
->lcla_pool
;
823 struct d40_log_lli_bidir
*lli
= &desc
->lli_log
;
824 int lli_current
= desc
->lli_current
;
825 int lli_len
= desc
->lli_len
;
826 bool cyclic
= desc
->cyclic
;
827 int curr_lcla
= -EINVAL
;
829 bool use_esram_lcla
= chan
->base
->plat_data
->use_esram_lcla
;
833 * We may have partially running cyclic transfers, in case we did't get
834 * enough LCLA entries.
836 linkback
= cyclic
&& lli_current
== 0;
839 * For linkback, we need one LCLA even with only one link, because we
840 * can't link back to the one in LCPA space
842 if (linkback
|| (lli_len
- lli_current
> 1)) {
844 * If the channel is expected to use only soft_lli don't
845 * allocate a lcla. This is to avoid a HW issue that exists
846 * in some controller during a peripheral to memory transfer
847 * that uses linked lists.
849 if (!(chan
->phy_chan
->use_soft_lli
&&
850 chan
->dma_cfg
.dir
== DMA_DEV_TO_MEM
))
851 curr_lcla
= d40_lcla_alloc_one(chan
, desc
);
853 first_lcla
= curr_lcla
;
857 * For linkback, we normally load the LCPA in the loop since we need to
858 * link it to the second LCLA and not the first. However, if we
859 * couldn't even get a first LCLA, then we have to run in LCPA and
862 if (!linkback
|| curr_lcla
== -EINVAL
) {
863 unsigned int flags
= 0;
865 if (curr_lcla
== -EINVAL
)
866 flags
|= LLI_TERM_INT
;
868 d40_log_lli_lcpa_write(chan
->lcpa
,
869 &lli
->dst
[lli_current
],
870 &lli
->src
[lli_current
],
879 for (; lli_current
< lli_len
; lli_current
++) {
880 unsigned int lcla_offset
= chan
->phy_chan
->num
* 1024 +
882 struct d40_log_lli
*lcla
= pool
->base
+ lcla_offset
;
883 unsigned int flags
= 0;
886 if (lli_current
+ 1 < lli_len
)
887 next_lcla
= d40_lcla_alloc_one(chan
, desc
);
889 next_lcla
= linkback
? first_lcla
: -EINVAL
;
891 if (cyclic
|| next_lcla
== -EINVAL
)
892 flags
|= LLI_TERM_INT
;
894 if (linkback
&& curr_lcla
== first_lcla
) {
895 /* First link goes in both LCPA and LCLA */
896 d40_log_lli_lcpa_write(chan
->lcpa
,
897 &lli
->dst
[lli_current
],
898 &lli
->src
[lli_current
],
903 * One unused LCLA in the cyclic case if the very first
906 d40_log_lli_lcla_write(lcla
,
907 &lli
->dst
[lli_current
],
908 &lli
->src
[lli_current
],
912 * Cache maintenance is not needed if lcla is
915 if (!use_esram_lcla
) {
916 dma_sync_single_range_for_device(chan
->base
->dev
,
917 pool
->dma_addr
, lcla_offset
,
918 2 * sizeof(struct d40_log_lli
),
921 curr_lcla
= next_lcla
;
923 if (curr_lcla
== -EINVAL
|| curr_lcla
== first_lcla
) {
930 desc
->lli_current
= lli_current
;
933 static void d40_desc_load(struct d40_chan
*d40c
, struct d40_desc
*d40d
)
935 if (chan_is_physical(d40c
)) {
936 d40_phy_lli_load(d40c
, d40d
);
937 d40d
->lli_current
= d40d
->lli_len
;
939 d40_log_lli_to_lcxa(d40c
, d40d
);
942 static struct d40_desc
*d40_first_active_get(struct d40_chan
*d40c
)
946 if (list_empty(&d40c
->active
))
949 d
= list_first_entry(&d40c
->active
,
955 /* remove desc from current queue and add it to the pending_queue */
956 static void d40_desc_queue(struct d40_chan
*d40c
, struct d40_desc
*desc
)
958 d40_desc_remove(desc
);
959 desc
->is_in_client_list
= false;
960 list_add_tail(&desc
->node
, &d40c
->pending_queue
);
963 static struct d40_desc
*d40_first_pending(struct d40_chan
*d40c
)
967 if (list_empty(&d40c
->pending_queue
))
970 d
= list_first_entry(&d40c
->pending_queue
,
976 static struct d40_desc
*d40_first_queued(struct d40_chan
*d40c
)
980 if (list_empty(&d40c
->queue
))
983 d
= list_first_entry(&d40c
->queue
,
989 static struct d40_desc
*d40_first_done(struct d40_chan
*d40c
)
991 if (list_empty(&d40c
->done
))
994 return list_first_entry(&d40c
->done
, struct d40_desc
, node
);
997 static int d40_psize_2_burst_size(bool is_log
, int psize
)
1000 if (psize
== STEDMA40_PSIZE_LOG_1
)
1003 if (psize
== STEDMA40_PSIZE_PHY_1
)
1011 * The dma only supports transmitting packages up to
1012 * STEDMA40_MAX_SEG_SIZE * data_width, where data_width is stored in Bytes.
1014 * Calculate the total number of dma elements required to send the entire sg list.
1016 static int d40_size_2_dmalen(int size
, u32 data_width1
, u32 data_width2
)
1019 u32 max_w
= max(data_width1
, data_width2
);
1020 u32 min_w
= min(data_width1
, data_width2
);
1021 u32 seg_max
= ALIGN(STEDMA40_MAX_SEG_SIZE
* min_w
, max_w
);
1023 if (seg_max
> STEDMA40_MAX_SEG_SIZE
)
1026 if (!IS_ALIGNED(size
, max_w
))
1029 if (size
<= seg_max
)
1032 dmalen
= size
/ seg_max
;
1033 if (dmalen
* seg_max
< size
)
1039 static int d40_sg_2_dmalen(struct scatterlist
*sgl
, int sg_len
,
1040 u32 data_width1
, u32 data_width2
)
1042 struct scatterlist
*sg
;
1047 for_each_sg(sgl
, sg
, sg_len
, i
) {
1048 ret
= d40_size_2_dmalen(sg_dma_len(sg
),
1049 data_width1
, data_width2
);
1057 static int __d40_execute_command_phy(struct d40_chan
*d40c
,
1058 enum d40_command command
)
1062 void __iomem
*active_reg
;
1064 unsigned long flags
;
1067 if (command
== D40_DMA_STOP
) {
1068 ret
= __d40_execute_command_phy(d40c
, D40_DMA_SUSPEND_REQ
);
1073 spin_lock_irqsave(&d40c
->base
->execmd_lock
, flags
);
1075 if (d40c
->phy_chan
->num
% 2 == 0)
1076 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
1078 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
1080 if (command
== D40_DMA_SUSPEND_REQ
) {
1081 status
= (readl(active_reg
) &
1082 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1083 D40_CHAN_POS(d40c
->phy_chan
->num
);
1085 if (status
== D40_DMA_SUSPENDED
|| status
== D40_DMA_STOP
)
1089 wmask
= 0xffffffff & ~(D40_CHAN_POS_MASK(d40c
->phy_chan
->num
));
1090 writel(wmask
| (command
<< D40_CHAN_POS(d40c
->phy_chan
->num
)),
1093 if (command
== D40_DMA_SUSPEND_REQ
) {
1095 for (i
= 0 ; i
< D40_SUSPEND_MAX_IT
; i
++) {
1096 status
= (readl(active_reg
) &
1097 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1098 D40_CHAN_POS(d40c
->phy_chan
->num
);
1102 * Reduce the number of bus accesses while
1103 * waiting for the DMA to suspend.
1107 if (status
== D40_DMA_STOP
||
1108 status
== D40_DMA_SUSPENDED
)
1112 if (i
== D40_SUSPEND_MAX_IT
) {
1114 "unable to suspend the chl %d (log: %d) status %x\n",
1115 d40c
->phy_chan
->num
, d40c
->log_num
,
1123 spin_unlock_irqrestore(&d40c
->base
->execmd_lock
, flags
);
1127 static void d40_term_all(struct d40_chan
*d40c
)
1129 struct d40_desc
*d40d
;
1130 struct d40_desc
*_d
;
1132 /* Release completed descriptors */
1133 while ((d40d
= d40_first_done(d40c
))) {
1134 d40_desc_remove(d40d
);
1135 d40_desc_free(d40c
, d40d
);
1138 /* Release active descriptors */
1139 while ((d40d
= d40_first_active_get(d40c
))) {
1140 d40_desc_remove(d40d
);
1141 d40_desc_free(d40c
, d40d
);
1144 /* Release queued descriptors waiting for transfer */
1145 while ((d40d
= d40_first_queued(d40c
))) {
1146 d40_desc_remove(d40d
);
1147 d40_desc_free(d40c
, d40d
);
1150 /* Release pending descriptors */
1151 while ((d40d
= d40_first_pending(d40c
))) {
1152 d40_desc_remove(d40d
);
1153 d40_desc_free(d40c
, d40d
);
1156 /* Release client owned descriptors */
1157 if (!list_empty(&d40c
->client
))
1158 list_for_each_entry_safe(d40d
, _d
, &d40c
->client
, node
) {
1159 d40_desc_remove(d40d
);
1160 d40_desc_free(d40c
, d40d
);
1163 /* Release descriptors in prepare queue */
1164 if (!list_empty(&d40c
->prepare_queue
))
1165 list_for_each_entry_safe(d40d
, _d
,
1166 &d40c
->prepare_queue
, node
) {
1167 d40_desc_remove(d40d
);
1168 d40_desc_free(d40c
, d40d
);
1171 d40c
->pending_tx
= 0;
1174 static void __d40_config_set_event(struct d40_chan
*d40c
,
1175 enum d40_events event_type
, u32 event
,
1178 void __iomem
*addr
= chan_base(d40c
) + reg
;
1182 switch (event_type
) {
1184 case D40_DEACTIVATE_EVENTLINE
:
1186 writel((D40_DEACTIVATE_EVENTLINE
<< D40_EVENTLINE_POS(event
))
1187 | ~D40_EVENTLINE_MASK(event
), addr
);
1190 case D40_SUSPEND_REQ_EVENTLINE
:
1191 status
= (readl(addr
) & D40_EVENTLINE_MASK(event
)) >>
1192 D40_EVENTLINE_POS(event
);
1194 if (status
== D40_DEACTIVATE_EVENTLINE
||
1195 status
== D40_SUSPEND_REQ_EVENTLINE
)
1198 writel((D40_SUSPEND_REQ_EVENTLINE
<< D40_EVENTLINE_POS(event
))
1199 | ~D40_EVENTLINE_MASK(event
), addr
);
1201 for (tries
= 0 ; tries
< D40_SUSPEND_MAX_IT
; tries
++) {
1203 status
= (readl(addr
) & D40_EVENTLINE_MASK(event
)) >>
1204 D40_EVENTLINE_POS(event
);
1208 * Reduce the number of bus accesses while
1209 * waiting for the DMA to suspend.
1213 if (status
== D40_DEACTIVATE_EVENTLINE
)
1217 if (tries
== D40_SUSPEND_MAX_IT
) {
1219 "unable to stop the event_line chl %d (log: %d)"
1220 "status %x\n", d40c
->phy_chan
->num
,
1221 d40c
->log_num
, status
);
1225 case D40_ACTIVATE_EVENTLINE
:
1227 * The hardware sometimes doesn't register the enable when src and dst
1228 * event lines are active on the same logical channel. Retry to ensure
1229 * it does. Usually only one retry is sufficient.
1233 writel((D40_ACTIVATE_EVENTLINE
<<
1234 D40_EVENTLINE_POS(event
)) |
1235 ~D40_EVENTLINE_MASK(event
), addr
);
1237 if (readl(addr
) & D40_EVENTLINE_MASK(event
))
1242 dev_dbg(chan2dev(d40c
),
1243 "[%s] workaround enable S%cLNK (%d tries)\n",
1244 __func__
, reg
== D40_CHAN_REG_SSLNK
? 'S' : 'D',
1250 case D40_ROUND_EVENTLINE
:
1257 static void d40_config_set_event(struct d40_chan
*d40c
,
1258 enum d40_events event_type
)
1260 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dev_type
);
1262 /* Enable event line connected to device (or memcpy) */
1263 if ((d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) ||
1264 (d40c
->dma_cfg
.dir
== DMA_DEV_TO_DEV
))
1265 __d40_config_set_event(d40c
, event_type
, event
,
1266 D40_CHAN_REG_SSLNK
);
1268 if (d40c
->dma_cfg
.dir
!= DMA_DEV_TO_MEM
)
1269 __d40_config_set_event(d40c
, event_type
, event
,
1270 D40_CHAN_REG_SDLNK
);
1273 static u32
d40_chan_has_events(struct d40_chan
*d40c
)
1275 void __iomem
*chanbase
= chan_base(d40c
);
1278 val
= readl(chanbase
+ D40_CHAN_REG_SSLNK
);
1279 val
|= readl(chanbase
+ D40_CHAN_REG_SDLNK
);
1285 __d40_execute_command_log(struct d40_chan
*d40c
, enum d40_command command
)
1287 unsigned long flags
;
1290 void __iomem
*active_reg
;
1292 if (d40c
->phy_chan
->num
% 2 == 0)
1293 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
1295 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
1298 spin_lock_irqsave(&d40c
->phy_chan
->lock
, flags
);
1302 case D40_DMA_SUSPEND_REQ
:
1304 active_status
= (readl(active_reg
) &
1305 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
1306 D40_CHAN_POS(d40c
->phy_chan
->num
);
1308 if (active_status
== D40_DMA_RUN
)
1309 d40_config_set_event(d40c
, D40_SUSPEND_REQ_EVENTLINE
);
1311 d40_config_set_event(d40c
, D40_DEACTIVATE_EVENTLINE
);
1313 if (!d40_chan_has_events(d40c
) && (command
== D40_DMA_STOP
))
1314 ret
= __d40_execute_command_phy(d40c
, command
);
1320 d40_config_set_event(d40c
, D40_ACTIVATE_EVENTLINE
);
1321 ret
= __d40_execute_command_phy(d40c
, command
);
1324 case D40_DMA_SUSPENDED
:
1329 spin_unlock_irqrestore(&d40c
->phy_chan
->lock
, flags
);
1333 static int d40_channel_execute_command(struct d40_chan
*d40c
,
1334 enum d40_command command
)
1336 if (chan_is_logical(d40c
))
1337 return __d40_execute_command_log(d40c
, command
);
1339 return __d40_execute_command_phy(d40c
, command
);
1342 static u32
d40_get_prmo(struct d40_chan
*d40c
)
1344 static const unsigned int phy_map
[] = {
1345 [STEDMA40_PCHAN_BASIC_MODE
]
1346 = D40_DREG_PRMO_PCHAN_BASIC
,
1347 [STEDMA40_PCHAN_MODULO_MODE
]
1348 = D40_DREG_PRMO_PCHAN_MODULO
,
1349 [STEDMA40_PCHAN_DOUBLE_DST_MODE
]
1350 = D40_DREG_PRMO_PCHAN_DOUBLE_DST
,
1352 static const unsigned int log_map
[] = {
1353 [STEDMA40_LCHAN_SRC_PHY_DST_LOG
]
1354 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG
,
1355 [STEDMA40_LCHAN_SRC_LOG_DST_PHY
]
1356 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY
,
1357 [STEDMA40_LCHAN_SRC_LOG_DST_LOG
]
1358 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG
,
1361 if (chan_is_physical(d40c
))
1362 return phy_map
[d40c
->dma_cfg
.mode_opt
];
1364 return log_map
[d40c
->dma_cfg
.mode_opt
];
1367 static void d40_config_write(struct d40_chan
*d40c
)
1372 /* Odd addresses are even addresses + 4 */
1373 addr_base
= (d40c
->phy_chan
->num
% 2) * 4;
1374 /* Setup channel mode to logical or physical */
1375 var
= ((u32
)(chan_is_logical(d40c
)) + 1) <<
1376 D40_CHAN_POS(d40c
->phy_chan
->num
);
1377 writel(var
, d40c
->base
->virtbase
+ D40_DREG_PRMSE
+ addr_base
);
1379 /* Setup operational mode option register */
1380 var
= d40_get_prmo(d40c
) << D40_CHAN_POS(d40c
->phy_chan
->num
);
1382 writel(var
, d40c
->base
->virtbase
+ D40_DREG_PRMOE
+ addr_base
);
1384 if (chan_is_logical(d40c
)) {
1385 int lidx
= (d40c
->phy_chan
->num
<< D40_SREG_ELEM_LOG_LIDX_POS
)
1386 & D40_SREG_ELEM_LOG_LIDX_MASK
;
1387 void __iomem
*chanbase
= chan_base(d40c
);
1389 /* Set default config for CFG reg */
1390 writel(d40c
->src_def_cfg
, chanbase
+ D40_CHAN_REG_SSCFG
);
1391 writel(d40c
->dst_def_cfg
, chanbase
+ D40_CHAN_REG_SDCFG
);
1393 /* Set LIDX for lcla */
1394 writel(lidx
, chanbase
+ D40_CHAN_REG_SSELT
);
1395 writel(lidx
, chanbase
+ D40_CHAN_REG_SDELT
);
1397 /* Clear LNK which will be used by d40_chan_has_events() */
1398 writel(0, chanbase
+ D40_CHAN_REG_SSLNK
);
1399 writel(0, chanbase
+ D40_CHAN_REG_SDLNK
);
1403 static u32
d40_residue(struct d40_chan
*d40c
)
1407 if (chan_is_logical(d40c
))
1408 num_elt
= (readl(&d40c
->lcpa
->lcsp2
) & D40_MEM_LCSP2_ECNT_MASK
)
1409 >> D40_MEM_LCSP2_ECNT_POS
;
1411 u32 val
= readl(chan_base(d40c
) + D40_CHAN_REG_SDELT
);
1412 num_elt
= (val
& D40_SREG_ELEM_PHY_ECNT_MASK
)
1413 >> D40_SREG_ELEM_PHY_ECNT_POS
;
1416 return num_elt
* d40c
->dma_cfg
.dst_info
.data_width
;
1419 static bool d40_tx_is_linked(struct d40_chan
*d40c
)
1423 if (chan_is_logical(d40c
))
1424 is_link
= readl(&d40c
->lcpa
->lcsp3
) & D40_MEM_LCSP3_DLOS_MASK
;
1426 is_link
= readl(chan_base(d40c
) + D40_CHAN_REG_SDLNK
)
1427 & D40_SREG_LNK_PHYS_LNK_MASK
;
1432 static int d40_pause(struct d40_chan
*d40c
)
1435 unsigned long flags
;
1440 spin_lock_irqsave(&d40c
->lock
, flags
);
1441 pm_runtime_get_sync(d40c
->base
->dev
);
1443 res
= d40_channel_execute_command(d40c
, D40_DMA_SUSPEND_REQ
);
1445 pm_runtime_mark_last_busy(d40c
->base
->dev
);
1446 pm_runtime_put_autosuspend(d40c
->base
->dev
);
1447 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1451 static int d40_resume(struct d40_chan
*d40c
)
1454 unsigned long flags
;
1459 spin_lock_irqsave(&d40c
->lock
, flags
);
1460 pm_runtime_get_sync(d40c
->base
->dev
);
1462 /* If bytes left to transfer or linked tx resume job */
1463 if (d40_residue(d40c
) || d40_tx_is_linked(d40c
))
1464 res
= d40_channel_execute_command(d40c
, D40_DMA_RUN
);
1466 pm_runtime_mark_last_busy(d40c
->base
->dev
);
1467 pm_runtime_put_autosuspend(d40c
->base
->dev
);
1468 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1472 static dma_cookie_t
d40_tx_submit(struct dma_async_tx_descriptor
*tx
)
1474 struct d40_chan
*d40c
= container_of(tx
->chan
,
1477 struct d40_desc
*d40d
= container_of(tx
, struct d40_desc
, txd
);
1478 unsigned long flags
;
1479 dma_cookie_t cookie
;
1481 spin_lock_irqsave(&d40c
->lock
, flags
);
1482 cookie
= dma_cookie_assign(tx
);
1483 d40_desc_queue(d40c
, d40d
);
1484 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1489 static int d40_start(struct d40_chan
*d40c
)
1491 return d40_channel_execute_command(d40c
, D40_DMA_RUN
);
1494 static struct d40_desc
*d40_queue_start(struct d40_chan
*d40c
)
1496 struct d40_desc
*d40d
;
1499 /* Start queued jobs, if any */
1500 d40d
= d40_first_queued(d40c
);
1505 pm_runtime_get_sync(d40c
->base
->dev
);
1508 /* Remove from queue */
1509 d40_desc_remove(d40d
);
1511 /* Add to active queue */
1512 d40_desc_submit(d40c
, d40d
);
1514 /* Initiate DMA job */
1515 d40_desc_load(d40c
, d40d
);
1518 err
= d40_start(d40c
);
1527 /* called from interrupt context */
1528 static void dma_tc_handle(struct d40_chan
*d40c
)
1530 struct d40_desc
*d40d
;
1532 /* Get first active entry from list */
1533 d40d
= d40_first_active_get(d40c
);
1540 * If this was a paritially loaded list, we need to reloaded
1541 * it, and only when the list is completed. We need to check
1542 * for done because the interrupt will hit for every link, and
1543 * not just the last one.
1545 if (d40d
->lli_current
< d40d
->lli_len
1546 && !d40_tx_is_linked(d40c
)
1547 && !d40_residue(d40c
)) {
1548 d40_lcla_free_all(d40c
, d40d
);
1549 d40_desc_load(d40c
, d40d
);
1550 (void) d40_start(d40c
);
1552 if (d40d
->lli_current
== d40d
->lli_len
)
1553 d40d
->lli_current
= 0;
1556 d40_lcla_free_all(d40c
, d40d
);
1558 if (d40d
->lli_current
< d40d
->lli_len
) {
1559 d40_desc_load(d40c
, d40d
);
1561 (void) d40_start(d40c
);
1565 if (d40_queue_start(d40c
) == NULL
) {
1568 pm_runtime_mark_last_busy(d40c
->base
->dev
);
1569 pm_runtime_put_autosuspend(d40c
->base
->dev
);
1572 d40_desc_remove(d40d
);
1573 d40_desc_done(d40c
, d40d
);
1577 tasklet_schedule(&d40c
->tasklet
);
1581 static void dma_tasklet(unsigned long data
)
1583 struct d40_chan
*d40c
= (struct d40_chan
*) data
;
1584 struct d40_desc
*d40d
;
1585 unsigned long flags
;
1586 bool callback_active
;
1587 dma_async_tx_callback callback
;
1588 void *callback_param
;
1590 spin_lock_irqsave(&d40c
->lock
, flags
);
1592 /* Get first entry from the done list */
1593 d40d
= d40_first_done(d40c
);
1595 /* Check if we have reached here for cyclic job */
1596 d40d
= d40_first_active_get(d40c
);
1597 if (d40d
== NULL
|| !d40d
->cyclic
)
1602 dma_cookie_complete(&d40d
->txd
);
1605 * If terminating a channel pending_tx is set to zero.
1606 * This prevents any finished active jobs to return to the client.
1608 if (d40c
->pending_tx
== 0) {
1609 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1613 /* Callback to client */
1614 callback_active
= !!(d40d
->txd
.flags
& DMA_PREP_INTERRUPT
);
1615 callback
= d40d
->txd
.callback
;
1616 callback_param
= d40d
->txd
.callback_param
;
1618 if (!d40d
->cyclic
) {
1619 if (async_tx_test_ack(&d40d
->txd
)) {
1620 d40_desc_remove(d40d
);
1621 d40_desc_free(d40c
, d40d
);
1622 } else if (!d40d
->is_in_client_list
) {
1623 d40_desc_remove(d40d
);
1624 d40_lcla_free_all(d40c
, d40d
);
1625 list_add_tail(&d40d
->node
, &d40c
->client
);
1626 d40d
->is_in_client_list
= true;
1632 if (d40c
->pending_tx
)
1633 tasklet_schedule(&d40c
->tasklet
);
1635 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1637 if (callback_active
&& callback
)
1638 callback(callback_param
);
1643 /* Rescue manouver if receiving double interrupts */
1644 if (d40c
->pending_tx
> 0)
1646 spin_unlock_irqrestore(&d40c
->lock
, flags
);
1649 static irqreturn_t
d40_handle_interrupt(int irq
, void *data
)
1655 struct d40_chan
*d40c
;
1656 unsigned long flags
;
1657 struct d40_base
*base
= data
;
1658 u32 regs
[base
->gen_dmac
.il_size
];
1659 struct d40_interrupt_lookup
*il
= base
->gen_dmac
.il
;
1660 u32 il_size
= base
->gen_dmac
.il_size
;
1662 spin_lock_irqsave(&base
->interrupt_lock
, flags
);
1664 /* Read interrupt status of both logical and physical channels */
1665 for (i
= 0; i
< il_size
; i
++)
1666 regs
[i
] = readl(base
->virtbase
+ il
[i
].src
);
1670 chan
= find_next_bit((unsigned long *)regs
,
1671 BITS_PER_LONG
* il_size
, chan
+ 1);
1673 /* No more set bits found? */
1674 if (chan
== BITS_PER_LONG
* il_size
)
1677 row
= chan
/ BITS_PER_LONG
;
1678 idx
= chan
& (BITS_PER_LONG
- 1);
1680 if (il
[row
].offset
== D40_PHY_CHAN
)
1681 d40c
= base
->lookup_phy_chans
[idx
];
1683 d40c
= base
->lookup_log_chans
[il
[row
].offset
+ idx
];
1687 * No error because this can happen if something else
1688 * in the system is using the channel.
1694 writel(BIT(idx
), base
->virtbase
+ il
[row
].clr
);
1696 spin_lock(&d40c
->lock
);
1698 if (!il
[row
].is_error
)
1699 dma_tc_handle(d40c
);
1701 d40_err(base
->dev
, "IRQ chan: %ld offset %d idx %d\n",
1702 chan
, il
[row
].offset
, idx
);
1704 spin_unlock(&d40c
->lock
);
1707 spin_unlock_irqrestore(&base
->interrupt_lock
, flags
);
1712 static int d40_validate_conf(struct d40_chan
*d40c
,
1713 struct stedma40_chan_cfg
*conf
)
1716 bool is_log
= conf
->mode
== STEDMA40_MODE_LOGICAL
;
1719 chan_err(d40c
, "Invalid direction.\n");
1723 if ((is_log
&& conf
->dev_type
> d40c
->base
->num_log_chans
) ||
1724 (!is_log
&& conf
->dev_type
> d40c
->base
->num_phy_chans
) ||
1725 (conf
->dev_type
< 0)) {
1726 chan_err(d40c
, "Invalid device type (%d)\n", conf
->dev_type
);
1730 if (conf
->dir
== DMA_DEV_TO_DEV
) {
1732 * DMAC HW supports it. Will be added to this driver,
1733 * in case any dma client requires it.
1735 chan_err(d40c
, "periph to periph not supported\n");
1739 if (d40_psize_2_burst_size(is_log
, conf
->src_info
.psize
) *
1740 conf
->src_info
.data_width
!=
1741 d40_psize_2_burst_size(is_log
, conf
->dst_info
.psize
) *
1742 conf
->dst_info
.data_width
) {
1744 * The DMAC hardware only supports
1745 * src (burst x width) == dst (burst x width)
1748 chan_err(d40c
, "src (burst x width) != dst (burst x width)\n");
1755 static bool d40_alloc_mask_set(struct d40_phy_res
*phy
,
1756 bool is_src
, int log_event_line
, bool is_log
,
1759 unsigned long flags
;
1760 spin_lock_irqsave(&phy
->lock
, flags
);
1762 *first_user
= ((phy
->allocated_src
| phy
->allocated_dst
)
1766 /* Physical interrupts are masked per physical full channel */
1767 if (phy
->allocated_src
== D40_ALLOC_FREE
&&
1768 phy
->allocated_dst
== D40_ALLOC_FREE
) {
1769 phy
->allocated_dst
= D40_ALLOC_PHY
;
1770 phy
->allocated_src
= D40_ALLOC_PHY
;
1776 /* Logical channel */
1778 if (phy
->allocated_src
== D40_ALLOC_PHY
)
1781 if (phy
->allocated_src
== D40_ALLOC_FREE
)
1782 phy
->allocated_src
= D40_ALLOC_LOG_FREE
;
1784 if (!(phy
->allocated_src
& BIT(log_event_line
))) {
1785 phy
->allocated_src
|= BIT(log_event_line
);
1790 if (phy
->allocated_dst
== D40_ALLOC_PHY
)
1793 if (phy
->allocated_dst
== D40_ALLOC_FREE
)
1794 phy
->allocated_dst
= D40_ALLOC_LOG_FREE
;
1796 if (!(phy
->allocated_dst
& BIT(log_event_line
))) {
1797 phy
->allocated_dst
|= BIT(log_event_line
);
1804 spin_unlock_irqrestore(&phy
->lock
, flags
);
1807 spin_unlock_irqrestore(&phy
->lock
, flags
);
1811 static bool d40_alloc_mask_free(struct d40_phy_res
*phy
, bool is_src
,
1814 unsigned long flags
;
1815 bool is_free
= false;
1817 spin_lock_irqsave(&phy
->lock
, flags
);
1818 if (!log_event_line
) {
1819 phy
->allocated_dst
= D40_ALLOC_FREE
;
1820 phy
->allocated_src
= D40_ALLOC_FREE
;
1825 /* Logical channel */
1827 phy
->allocated_src
&= ~BIT(log_event_line
);
1828 if (phy
->allocated_src
== D40_ALLOC_LOG_FREE
)
1829 phy
->allocated_src
= D40_ALLOC_FREE
;
1831 phy
->allocated_dst
&= ~BIT(log_event_line
);
1832 if (phy
->allocated_dst
== D40_ALLOC_LOG_FREE
)
1833 phy
->allocated_dst
= D40_ALLOC_FREE
;
1836 is_free
= ((phy
->allocated_src
| phy
->allocated_dst
) ==
1840 spin_unlock_irqrestore(&phy
->lock
, flags
);
1845 static int d40_allocate_channel(struct d40_chan
*d40c
, bool *first_phy_user
)
1847 int dev_type
= d40c
->dma_cfg
.dev_type
;
1850 struct d40_phy_res
*phys
;
1856 bool is_log
= d40c
->dma_cfg
.mode
== STEDMA40_MODE_LOGICAL
;
1858 phys
= d40c
->base
->phy_res
;
1859 num_phy_chans
= d40c
->base
->num_phy_chans
;
1861 if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) {
1862 log_num
= 2 * dev_type
;
1864 } else if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
||
1865 d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
) {
1866 /* dst event lines are used for logical memcpy */
1867 log_num
= 2 * dev_type
+ 1;
1872 event_group
= D40_TYPE_TO_GROUP(dev_type
);
1873 event_line
= D40_TYPE_TO_EVENT(dev_type
);
1876 if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
) {
1877 /* Find physical half channel */
1878 if (d40c
->dma_cfg
.use_fixed_channel
) {
1879 i
= d40c
->dma_cfg
.phy_channel
;
1880 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1885 for (i
= 0; i
< num_phy_chans
; i
++) {
1886 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1893 for (j
= 0; j
< d40c
->base
->num_phy_chans
; j
+= 8) {
1894 int phy_num
= j
+ event_group
* 2;
1895 for (i
= phy_num
; i
< phy_num
+ 2; i
++) {
1896 if (d40_alloc_mask_set(&phys
[i
],
1906 d40c
->phy_chan
= &phys
[i
];
1907 d40c
->log_num
= D40_PHY_CHAN
;
1913 /* Find logical channel */
1914 for (j
= 0; j
< d40c
->base
->num_phy_chans
; j
+= 8) {
1915 int phy_num
= j
+ event_group
* 2;
1917 if (d40c
->dma_cfg
.use_fixed_channel
) {
1918 i
= d40c
->dma_cfg
.phy_channel
;
1920 if ((i
!= phy_num
) && (i
!= phy_num
+ 1)) {
1921 dev_err(chan2dev(d40c
),
1922 "invalid fixed phy channel %d\n", i
);
1926 if (d40_alloc_mask_set(&phys
[i
], is_src
, event_line
,
1927 is_log
, first_phy_user
))
1930 dev_err(chan2dev(d40c
),
1931 "could not allocate fixed phy channel %d\n", i
);
1936 * Spread logical channels across all available physical rather
1937 * than pack every logical channel at the first available phy
1941 for (i
= phy_num
; i
< phy_num
+ 2; i
++) {
1942 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1948 for (i
= phy_num
+ 1; i
>= phy_num
; i
--) {
1949 if (d40_alloc_mask_set(&phys
[i
], is_src
,
1959 d40c
->phy_chan
= &phys
[i
];
1960 d40c
->log_num
= log_num
;
1964 d40c
->base
->lookup_log_chans
[d40c
->log_num
] = d40c
;
1966 d40c
->base
->lookup_phy_chans
[d40c
->phy_chan
->num
] = d40c
;
1972 static int d40_config_memcpy(struct d40_chan
*d40c
)
1974 dma_cap_mask_t cap
= d40c
->chan
.device
->cap_mask
;
1976 if (dma_has_cap(DMA_MEMCPY
, cap
) && !dma_has_cap(DMA_SLAVE
, cap
)) {
1977 d40c
->dma_cfg
= dma40_memcpy_conf_log
;
1978 d40c
->dma_cfg
.dev_type
= dma40_memcpy_channels
[d40c
->chan
.chan_id
];
1980 d40_log_cfg(&d40c
->dma_cfg
,
1981 &d40c
->log_def
.lcsp1
, &d40c
->log_def
.lcsp3
);
1983 } else if (dma_has_cap(DMA_MEMCPY
, cap
) &&
1984 dma_has_cap(DMA_SLAVE
, cap
)) {
1985 d40c
->dma_cfg
= dma40_memcpy_conf_phy
;
1987 /* Generate interrrupt at end of transfer or relink. */
1988 d40c
->dst_def_cfg
|= BIT(D40_SREG_CFG_TIM_POS
);
1990 /* Generate interrupt on error. */
1991 d40c
->src_def_cfg
|= BIT(D40_SREG_CFG_EIM_POS
);
1992 d40c
->dst_def_cfg
|= BIT(D40_SREG_CFG_EIM_POS
);
1995 chan_err(d40c
, "No memcpy\n");
2002 static int d40_free_dma(struct d40_chan
*d40c
)
2006 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dev_type
);
2007 struct d40_phy_res
*phy
= d40c
->phy_chan
;
2010 /* Terminate all queued and active transfers */
2014 chan_err(d40c
, "phy == null\n");
2018 if (phy
->allocated_src
== D40_ALLOC_FREE
&&
2019 phy
->allocated_dst
== D40_ALLOC_FREE
) {
2020 chan_err(d40c
, "channel already free\n");
2024 if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
||
2025 d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
)
2027 else if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
)
2030 chan_err(d40c
, "Unknown direction\n");
2034 pm_runtime_get_sync(d40c
->base
->dev
);
2035 res
= d40_channel_execute_command(d40c
, D40_DMA_STOP
);
2037 chan_err(d40c
, "stop failed\n");
2041 d40_alloc_mask_free(phy
, is_src
, chan_is_logical(d40c
) ? event
: 0);
2043 if (chan_is_logical(d40c
))
2044 d40c
->base
->lookup_log_chans
[d40c
->log_num
] = NULL
;
2046 d40c
->base
->lookup_phy_chans
[phy
->num
] = NULL
;
2049 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2050 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2054 d40c
->phy_chan
= NULL
;
2055 d40c
->configured
= false;
2058 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2059 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2063 static bool d40_is_paused(struct d40_chan
*d40c
)
2065 void __iomem
*chanbase
= chan_base(d40c
);
2066 bool is_paused
= false;
2067 unsigned long flags
;
2068 void __iomem
*active_reg
;
2070 u32 event
= D40_TYPE_TO_EVENT(d40c
->dma_cfg
.dev_type
);
2072 spin_lock_irqsave(&d40c
->lock
, flags
);
2074 if (chan_is_physical(d40c
)) {
2075 if (d40c
->phy_chan
->num
% 2 == 0)
2076 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVE
;
2078 active_reg
= d40c
->base
->virtbase
+ D40_DREG_ACTIVO
;
2080 status
= (readl(active_reg
) &
2081 D40_CHAN_POS_MASK(d40c
->phy_chan
->num
)) >>
2082 D40_CHAN_POS(d40c
->phy_chan
->num
);
2083 if (status
== D40_DMA_SUSPENDED
|| status
== D40_DMA_STOP
)
2089 if (d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
||
2090 d40c
->dma_cfg
.dir
== DMA_MEM_TO_MEM
) {
2091 status
= readl(chanbase
+ D40_CHAN_REG_SDLNK
);
2092 } else if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) {
2093 status
= readl(chanbase
+ D40_CHAN_REG_SSLNK
);
2095 chan_err(d40c
, "Unknown direction\n");
2099 status
= (status
& D40_EVENTLINE_MASK(event
)) >>
2100 D40_EVENTLINE_POS(event
);
2102 if (status
!= D40_DMA_RUN
)
2105 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2110 static u32
stedma40_residue(struct dma_chan
*chan
)
2112 struct d40_chan
*d40c
=
2113 container_of(chan
, struct d40_chan
, chan
);
2115 unsigned long flags
;
2117 spin_lock_irqsave(&d40c
->lock
, flags
);
2118 bytes_left
= d40_residue(d40c
);
2119 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2125 d40_prep_sg_log(struct d40_chan
*chan
, struct d40_desc
*desc
,
2126 struct scatterlist
*sg_src
, struct scatterlist
*sg_dst
,
2127 unsigned int sg_len
, dma_addr_t src_dev_addr
,
2128 dma_addr_t dst_dev_addr
)
2130 struct stedma40_chan_cfg
*cfg
= &chan
->dma_cfg
;
2131 struct stedma40_half_channel_info
*src_info
= &cfg
->src_info
;
2132 struct stedma40_half_channel_info
*dst_info
= &cfg
->dst_info
;
2135 ret
= d40_log_sg_to_lli(sg_src
, sg_len
,
2138 chan
->log_def
.lcsp1
,
2139 src_info
->data_width
,
2140 dst_info
->data_width
);
2142 ret
= d40_log_sg_to_lli(sg_dst
, sg_len
,
2145 chan
->log_def
.lcsp3
,
2146 dst_info
->data_width
,
2147 src_info
->data_width
);
2149 return ret
< 0 ? ret
: 0;
2153 d40_prep_sg_phy(struct d40_chan
*chan
, struct d40_desc
*desc
,
2154 struct scatterlist
*sg_src
, struct scatterlist
*sg_dst
,
2155 unsigned int sg_len
, dma_addr_t src_dev_addr
,
2156 dma_addr_t dst_dev_addr
)
2158 struct stedma40_chan_cfg
*cfg
= &chan
->dma_cfg
;
2159 struct stedma40_half_channel_info
*src_info
= &cfg
->src_info
;
2160 struct stedma40_half_channel_info
*dst_info
= &cfg
->dst_info
;
2161 unsigned long flags
= 0;
2165 flags
|= LLI_CYCLIC
| LLI_TERM_INT
;
2167 ret
= d40_phy_sg_to_lli(sg_src
, sg_len
, src_dev_addr
,
2169 virt_to_phys(desc
->lli_phy
.src
),
2171 src_info
, dst_info
, flags
);
2173 ret
= d40_phy_sg_to_lli(sg_dst
, sg_len
, dst_dev_addr
,
2175 virt_to_phys(desc
->lli_phy
.dst
),
2177 dst_info
, src_info
, flags
);
2179 dma_sync_single_for_device(chan
->base
->dev
, desc
->lli_pool
.dma_addr
,
2180 desc
->lli_pool
.size
, DMA_TO_DEVICE
);
2182 return ret
< 0 ? ret
: 0;
2185 static struct d40_desc
*
2186 d40_prep_desc(struct d40_chan
*chan
, struct scatterlist
*sg
,
2187 unsigned int sg_len
, unsigned long dma_flags
)
2189 struct stedma40_chan_cfg
*cfg
= &chan
->dma_cfg
;
2190 struct d40_desc
*desc
;
2193 desc
= d40_desc_get(chan
);
2197 desc
->lli_len
= d40_sg_2_dmalen(sg
, sg_len
, cfg
->src_info
.data_width
,
2198 cfg
->dst_info
.data_width
);
2199 if (desc
->lli_len
< 0) {
2200 chan_err(chan
, "Unaligned size\n");
2204 ret
= d40_pool_lli_alloc(chan
, desc
, desc
->lli_len
);
2206 chan_err(chan
, "Could not allocate lli\n");
2210 desc
->lli_current
= 0;
2211 desc
->txd
.flags
= dma_flags
;
2212 desc
->txd
.tx_submit
= d40_tx_submit
;
2214 dma_async_tx_descriptor_init(&desc
->txd
, &chan
->chan
);
2219 d40_desc_free(chan
, desc
);
2223 static struct dma_async_tx_descriptor
*
2224 d40_prep_sg(struct dma_chan
*dchan
, struct scatterlist
*sg_src
,
2225 struct scatterlist
*sg_dst
, unsigned int sg_len
,
2226 enum dma_transfer_direction direction
, unsigned long dma_flags
)
2228 struct d40_chan
*chan
= container_of(dchan
, struct d40_chan
, chan
);
2229 dma_addr_t src_dev_addr
= 0;
2230 dma_addr_t dst_dev_addr
= 0;
2231 struct d40_desc
*desc
;
2232 unsigned long flags
;
2235 if (!chan
->phy_chan
) {
2236 chan_err(chan
, "Cannot prepare unallocated channel\n");
2240 spin_lock_irqsave(&chan
->lock
, flags
);
2242 desc
= d40_prep_desc(chan
, sg_src
, sg_len
, dma_flags
);
2246 if (sg_next(&sg_src
[sg_len
- 1]) == sg_src
)
2247 desc
->cyclic
= true;
2249 if (direction
== DMA_DEV_TO_MEM
)
2250 src_dev_addr
= chan
->runtime_addr
;
2251 else if (direction
== DMA_MEM_TO_DEV
)
2252 dst_dev_addr
= chan
->runtime_addr
;
2254 if (chan_is_logical(chan
))
2255 ret
= d40_prep_sg_log(chan
, desc
, sg_src
, sg_dst
,
2256 sg_len
, src_dev_addr
, dst_dev_addr
);
2258 ret
= d40_prep_sg_phy(chan
, desc
, sg_src
, sg_dst
,
2259 sg_len
, src_dev_addr
, dst_dev_addr
);
2262 chan_err(chan
, "Failed to prepare %s sg job: %d\n",
2263 chan_is_logical(chan
) ? "log" : "phy", ret
);
2268 * add descriptor to the prepare queue in order to be able
2269 * to free them later in terminate_all
2271 list_add_tail(&desc
->node
, &chan
->prepare_queue
);
2273 spin_unlock_irqrestore(&chan
->lock
, flags
);
2279 d40_desc_free(chan
, desc
);
2280 spin_unlock_irqrestore(&chan
->lock
, flags
);
2284 bool stedma40_filter(struct dma_chan
*chan
, void *data
)
2286 struct stedma40_chan_cfg
*info
= data
;
2287 struct d40_chan
*d40c
=
2288 container_of(chan
, struct d40_chan
, chan
);
2292 err
= d40_validate_conf(d40c
, info
);
2294 d40c
->dma_cfg
= *info
;
2296 err
= d40_config_memcpy(d40c
);
2299 d40c
->configured
= true;
2303 EXPORT_SYMBOL(stedma40_filter
);
2305 static void __d40_set_prio_rt(struct d40_chan
*d40c
, int dev_type
, bool src
)
2307 bool realtime
= d40c
->dma_cfg
.realtime
;
2308 bool highprio
= d40c
->dma_cfg
.high_priority
;
2310 u32 event
= D40_TYPE_TO_EVENT(dev_type
);
2311 u32 group
= D40_TYPE_TO_GROUP(dev_type
);
2312 u32 bit
= BIT(event
);
2314 struct d40_gen_dmac
*dmac
= &d40c
->base
->gen_dmac
;
2316 rtreg
= realtime
? dmac
->realtime_en
: dmac
->realtime_clear
;
2318 * Due to a hardware bug, in some cases a logical channel triggered by
2319 * a high priority destination event line can generate extra packet
2322 * The workaround is to not set the high priority level for the
2323 * destination event lines that trigger logical channels.
2325 if (!src
&& chan_is_logical(d40c
))
2328 prioreg
= highprio
? dmac
->high_prio_en
: dmac
->high_prio_clear
;
2330 /* Destination event lines are stored in the upper halfword */
2334 writel(bit
, d40c
->base
->virtbase
+ prioreg
+ group
* 4);
2335 writel(bit
, d40c
->base
->virtbase
+ rtreg
+ group
* 4);
2338 static void d40_set_prio_realtime(struct d40_chan
*d40c
)
2340 if (d40c
->base
->rev
< 3)
2343 if ((d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
) ||
2344 (d40c
->dma_cfg
.dir
== DMA_DEV_TO_DEV
))
2345 __d40_set_prio_rt(d40c
, d40c
->dma_cfg
.dev_type
, true);
2347 if ((d40c
->dma_cfg
.dir
== DMA_MEM_TO_DEV
) ||
2348 (d40c
->dma_cfg
.dir
== DMA_DEV_TO_DEV
))
2349 __d40_set_prio_rt(d40c
, d40c
->dma_cfg
.dev_type
, false);
2352 #define D40_DT_FLAGS_MODE(flags) ((flags >> 0) & 0x1)
2353 #define D40_DT_FLAGS_DIR(flags) ((flags >> 1) & 0x1)
2354 #define D40_DT_FLAGS_BIG_ENDIAN(flags) ((flags >> 2) & 0x1)
2355 #define D40_DT_FLAGS_FIXED_CHAN(flags) ((flags >> 3) & 0x1)
2356 #define D40_DT_FLAGS_HIGH_PRIO(flags) ((flags >> 4) & 0x1)
2358 static struct dma_chan
*d40_xlate(struct of_phandle_args
*dma_spec
,
2359 struct of_dma
*ofdma
)
2361 struct stedma40_chan_cfg cfg
;
2365 memset(&cfg
, 0, sizeof(struct stedma40_chan_cfg
));
2368 dma_cap_set(DMA_SLAVE
, cap
);
2370 cfg
.dev_type
= dma_spec
->args
[0];
2371 flags
= dma_spec
->args
[2];
2373 switch (D40_DT_FLAGS_MODE(flags
)) {
2374 case 0: cfg
.mode
= STEDMA40_MODE_LOGICAL
; break;
2375 case 1: cfg
.mode
= STEDMA40_MODE_PHYSICAL
; break;
2378 switch (D40_DT_FLAGS_DIR(flags
)) {
2380 cfg
.dir
= DMA_MEM_TO_DEV
;
2381 cfg
.dst_info
.big_endian
= D40_DT_FLAGS_BIG_ENDIAN(flags
);
2384 cfg
.dir
= DMA_DEV_TO_MEM
;
2385 cfg
.src_info
.big_endian
= D40_DT_FLAGS_BIG_ENDIAN(flags
);
2389 if (D40_DT_FLAGS_FIXED_CHAN(flags
)) {
2390 cfg
.phy_channel
= dma_spec
->args
[1];
2391 cfg
.use_fixed_channel
= true;
2394 if (D40_DT_FLAGS_HIGH_PRIO(flags
))
2395 cfg
.high_priority
= true;
2397 return dma_request_channel(cap
, stedma40_filter
, &cfg
);
2400 /* DMA ENGINE functions */
2401 static int d40_alloc_chan_resources(struct dma_chan
*chan
)
2404 unsigned long flags
;
2405 struct d40_chan
*d40c
=
2406 container_of(chan
, struct d40_chan
, chan
);
2408 spin_lock_irqsave(&d40c
->lock
, flags
);
2410 dma_cookie_init(chan
);
2412 /* If no dma configuration is set use default configuration (memcpy) */
2413 if (!d40c
->configured
) {
2414 err
= d40_config_memcpy(d40c
);
2416 chan_err(d40c
, "Failed to configure memcpy channel\n");
2421 err
= d40_allocate_channel(d40c
, &is_free_phy
);
2423 chan_err(d40c
, "Failed to allocate channel\n");
2424 d40c
->configured
= false;
2428 pm_runtime_get_sync(d40c
->base
->dev
);
2430 d40_set_prio_realtime(d40c
);
2432 if (chan_is_logical(d40c
)) {
2433 if (d40c
->dma_cfg
.dir
== DMA_DEV_TO_MEM
)
2434 d40c
->lcpa
= d40c
->base
->lcpa_base
+
2435 d40c
->dma_cfg
.dev_type
* D40_LCPA_CHAN_SIZE
;
2437 d40c
->lcpa
= d40c
->base
->lcpa_base
+
2438 d40c
->dma_cfg
.dev_type
*
2439 D40_LCPA_CHAN_SIZE
+ D40_LCPA_CHAN_DST_DELTA
;
2441 /* Unmask the Global Interrupt Mask. */
2442 d40c
->src_def_cfg
|= BIT(D40_SREG_CFG_LOG_GIM_POS
);
2443 d40c
->dst_def_cfg
|= BIT(D40_SREG_CFG_LOG_GIM_POS
);
2446 dev_dbg(chan2dev(d40c
), "allocated %s channel (phy %d%s)\n",
2447 chan_is_logical(d40c
) ? "logical" : "physical",
2448 d40c
->phy_chan
->num
,
2449 d40c
->dma_cfg
.use_fixed_channel
? ", fixed" : "");
2453 * Only write channel configuration to the DMA if the physical
2454 * resource is free. In case of multiple logical channels
2455 * on the same physical resource, only the first write is necessary.
2458 d40_config_write(d40c
);
2460 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2461 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2462 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2466 static void d40_free_chan_resources(struct dma_chan
*chan
)
2468 struct d40_chan
*d40c
=
2469 container_of(chan
, struct d40_chan
, chan
);
2471 unsigned long flags
;
2473 if (d40c
->phy_chan
== NULL
) {
2474 chan_err(d40c
, "Cannot free unallocated channel\n");
2478 spin_lock_irqsave(&d40c
->lock
, flags
);
2480 err
= d40_free_dma(d40c
);
2483 chan_err(d40c
, "Failed to free channel\n");
2484 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2487 static struct dma_async_tx_descriptor
*d40_prep_memcpy(struct dma_chan
*chan
,
2491 unsigned long dma_flags
)
2493 struct scatterlist dst_sg
;
2494 struct scatterlist src_sg
;
2496 sg_init_table(&dst_sg
, 1);
2497 sg_init_table(&src_sg
, 1);
2499 sg_dma_address(&dst_sg
) = dst
;
2500 sg_dma_address(&src_sg
) = src
;
2502 sg_dma_len(&dst_sg
) = size
;
2503 sg_dma_len(&src_sg
) = size
;
2505 return d40_prep_sg(chan
, &src_sg
, &dst_sg
, 1, DMA_NONE
, dma_flags
);
2508 static struct dma_async_tx_descriptor
*
2509 d40_prep_memcpy_sg(struct dma_chan
*chan
,
2510 struct scatterlist
*dst_sg
, unsigned int dst_nents
,
2511 struct scatterlist
*src_sg
, unsigned int src_nents
,
2512 unsigned long dma_flags
)
2514 if (dst_nents
!= src_nents
)
2517 return d40_prep_sg(chan
, src_sg
, dst_sg
, src_nents
, DMA_NONE
, dma_flags
);
2520 static struct dma_async_tx_descriptor
*
2521 d40_prep_slave_sg(struct dma_chan
*chan
, struct scatterlist
*sgl
,
2522 unsigned int sg_len
, enum dma_transfer_direction direction
,
2523 unsigned long dma_flags
, void *context
)
2525 if (!is_slave_direction(direction
))
2528 return d40_prep_sg(chan
, sgl
, sgl
, sg_len
, direction
, dma_flags
);
2531 static struct dma_async_tx_descriptor
*
2532 dma40_prep_dma_cyclic(struct dma_chan
*chan
, dma_addr_t dma_addr
,
2533 size_t buf_len
, size_t period_len
,
2534 enum dma_transfer_direction direction
, unsigned long flags
)
2536 unsigned int periods
= buf_len
/ period_len
;
2537 struct dma_async_tx_descriptor
*txd
;
2538 struct scatterlist
*sg
;
2541 sg
= kcalloc(periods
+ 1, sizeof(struct scatterlist
), GFP_NOWAIT
);
2545 for (i
= 0; i
< periods
; i
++) {
2546 sg_dma_address(&sg
[i
]) = dma_addr
;
2547 sg_dma_len(&sg
[i
]) = period_len
;
2548 dma_addr
+= period_len
;
2551 sg
[periods
].offset
= 0;
2552 sg_dma_len(&sg
[periods
]) = 0;
2553 sg
[periods
].page_link
=
2554 ((unsigned long)sg
| 0x01) & ~0x02;
2556 txd
= d40_prep_sg(chan
, sg
, sg
, periods
, direction
,
2557 DMA_PREP_INTERRUPT
);
2564 static enum dma_status
d40_tx_status(struct dma_chan
*chan
,
2565 dma_cookie_t cookie
,
2566 struct dma_tx_state
*txstate
)
2568 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2569 enum dma_status ret
;
2571 if (d40c
->phy_chan
== NULL
) {
2572 chan_err(d40c
, "Cannot read status of unallocated channel\n");
2576 ret
= dma_cookie_status(chan
, cookie
, txstate
);
2577 if (ret
!= DMA_COMPLETE
)
2578 dma_set_residue(txstate
, stedma40_residue(chan
));
2580 if (d40_is_paused(d40c
))
2586 static void d40_issue_pending(struct dma_chan
*chan
)
2588 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2589 unsigned long flags
;
2591 if (d40c
->phy_chan
== NULL
) {
2592 chan_err(d40c
, "Channel is not allocated!\n");
2596 spin_lock_irqsave(&d40c
->lock
, flags
);
2598 list_splice_tail_init(&d40c
->pending_queue
, &d40c
->queue
);
2600 /* Busy means that queued jobs are already being processed */
2602 (void) d40_queue_start(d40c
);
2604 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2607 static void d40_terminate_all(struct dma_chan
*chan
)
2609 unsigned long flags
;
2610 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2613 spin_lock_irqsave(&d40c
->lock
, flags
);
2615 pm_runtime_get_sync(d40c
->base
->dev
);
2616 ret
= d40_channel_execute_command(d40c
, D40_DMA_STOP
);
2618 chan_err(d40c
, "Failed to stop channel\n");
2621 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2622 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2624 pm_runtime_mark_last_busy(d40c
->base
->dev
);
2625 pm_runtime_put_autosuspend(d40c
->base
->dev
);
2629 spin_unlock_irqrestore(&d40c
->lock
, flags
);
2633 dma40_config_to_halfchannel(struct d40_chan
*d40c
,
2634 struct stedma40_half_channel_info
*info
,
2639 if (chan_is_logical(d40c
)) {
2641 psize
= STEDMA40_PSIZE_LOG_16
;
2642 else if (maxburst
>= 8)
2643 psize
= STEDMA40_PSIZE_LOG_8
;
2644 else if (maxburst
>= 4)
2645 psize
= STEDMA40_PSIZE_LOG_4
;
2647 psize
= STEDMA40_PSIZE_LOG_1
;
2650 psize
= STEDMA40_PSIZE_PHY_16
;
2651 else if (maxburst
>= 8)
2652 psize
= STEDMA40_PSIZE_PHY_8
;
2653 else if (maxburst
>= 4)
2654 psize
= STEDMA40_PSIZE_PHY_4
;
2656 psize
= STEDMA40_PSIZE_PHY_1
;
2659 info
->psize
= psize
;
2660 info
->flow_ctrl
= STEDMA40_NO_FLOW_CTRL
;
2665 /* Runtime reconfiguration extension */
2666 static int d40_set_runtime_config(struct dma_chan
*chan
,
2667 struct dma_slave_config
*config
)
2669 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2670 struct stedma40_chan_cfg
*cfg
= &d40c
->dma_cfg
;
2671 enum dma_slave_buswidth src_addr_width
, dst_addr_width
;
2672 dma_addr_t config_addr
;
2673 u32 src_maxburst
, dst_maxburst
;
2676 src_addr_width
= config
->src_addr_width
;
2677 src_maxburst
= config
->src_maxburst
;
2678 dst_addr_width
= config
->dst_addr_width
;
2679 dst_maxburst
= config
->dst_maxburst
;
2681 if (config
->direction
== DMA_DEV_TO_MEM
) {
2682 config_addr
= config
->src_addr
;
2684 if (cfg
->dir
!= DMA_DEV_TO_MEM
)
2685 dev_dbg(d40c
->base
->dev
,
2686 "channel was not configured for peripheral "
2687 "to memory transfer (%d) overriding\n",
2689 cfg
->dir
= DMA_DEV_TO_MEM
;
2691 /* Configure the memory side */
2692 if (dst_addr_width
== DMA_SLAVE_BUSWIDTH_UNDEFINED
)
2693 dst_addr_width
= src_addr_width
;
2694 if (dst_maxburst
== 0)
2695 dst_maxburst
= src_maxburst
;
2697 } else if (config
->direction
== DMA_MEM_TO_DEV
) {
2698 config_addr
= config
->dst_addr
;
2700 if (cfg
->dir
!= DMA_MEM_TO_DEV
)
2701 dev_dbg(d40c
->base
->dev
,
2702 "channel was not configured for memory "
2703 "to peripheral transfer (%d) overriding\n",
2705 cfg
->dir
= DMA_MEM_TO_DEV
;
2707 /* Configure the memory side */
2708 if (src_addr_width
== DMA_SLAVE_BUSWIDTH_UNDEFINED
)
2709 src_addr_width
= dst_addr_width
;
2710 if (src_maxburst
== 0)
2711 src_maxburst
= dst_maxburst
;
2713 dev_err(d40c
->base
->dev
,
2714 "unrecognized channel direction %d\n",
2719 if (config_addr
<= 0) {
2720 dev_err(d40c
->base
->dev
, "no address supplied\n");
2724 if (src_maxburst
* src_addr_width
!= dst_maxburst
* dst_addr_width
) {
2725 dev_err(d40c
->base
->dev
,
2726 "src/dst width/maxburst mismatch: %d*%d != %d*%d\n",
2734 if (src_maxburst
> 16) {
2736 dst_maxburst
= src_maxburst
* src_addr_width
/ dst_addr_width
;
2737 } else if (dst_maxburst
> 16) {
2739 src_maxburst
= dst_maxburst
* dst_addr_width
/ src_addr_width
;
2742 /* Only valid widths are; 1, 2, 4 and 8. */
2743 if (src_addr_width
<= DMA_SLAVE_BUSWIDTH_UNDEFINED
||
2744 src_addr_width
> DMA_SLAVE_BUSWIDTH_8_BYTES
||
2745 dst_addr_width
<= DMA_SLAVE_BUSWIDTH_UNDEFINED
||
2746 dst_addr_width
> DMA_SLAVE_BUSWIDTH_8_BYTES
||
2747 !is_power_of_2(src_addr_width
) ||
2748 !is_power_of_2(dst_addr_width
))
2751 cfg
->src_info
.data_width
= src_addr_width
;
2752 cfg
->dst_info
.data_width
= dst_addr_width
;
2754 ret
= dma40_config_to_halfchannel(d40c
, &cfg
->src_info
,
2759 ret
= dma40_config_to_halfchannel(d40c
, &cfg
->dst_info
,
2764 /* Fill in register values */
2765 if (chan_is_logical(d40c
))
2766 d40_log_cfg(cfg
, &d40c
->log_def
.lcsp1
, &d40c
->log_def
.lcsp3
);
2768 d40_phy_cfg(cfg
, &d40c
->src_def_cfg
, &d40c
->dst_def_cfg
);
2770 /* These settings will take precedence later */
2771 d40c
->runtime_addr
= config_addr
;
2772 d40c
->runtime_direction
= config
->direction
;
2773 dev_dbg(d40c
->base
->dev
,
2774 "configured channel %s for %s, data width %d/%d, "
2775 "maxburst %d/%d elements, LE, no flow control\n",
2776 dma_chan_name(chan
),
2777 (config
->direction
== DMA_DEV_TO_MEM
) ? "RX" : "TX",
2778 src_addr_width
, dst_addr_width
,
2779 src_maxburst
, dst_maxburst
);
2784 static int d40_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
2787 struct d40_chan
*d40c
= container_of(chan
, struct d40_chan
, chan
);
2789 if (d40c
->phy_chan
== NULL
) {
2790 chan_err(d40c
, "Channel is not allocated!\n");
2795 case DMA_TERMINATE_ALL
:
2796 d40_terminate_all(chan
);
2799 return d40_pause(d40c
);
2801 return d40_resume(d40c
);
2802 case DMA_SLAVE_CONFIG
:
2803 return d40_set_runtime_config(chan
,
2804 (struct dma_slave_config
*) arg
);
2809 /* Other commands are unimplemented */
2813 /* Initialization functions */
2815 static void __init
d40_chan_init(struct d40_base
*base
, struct dma_device
*dma
,
2816 struct d40_chan
*chans
, int offset
,
2820 struct d40_chan
*d40c
;
2822 INIT_LIST_HEAD(&dma
->channels
);
2824 for (i
= offset
; i
< offset
+ num_chans
; i
++) {
2827 d40c
->chan
.device
= dma
;
2829 spin_lock_init(&d40c
->lock
);
2831 d40c
->log_num
= D40_PHY_CHAN
;
2833 INIT_LIST_HEAD(&d40c
->done
);
2834 INIT_LIST_HEAD(&d40c
->active
);
2835 INIT_LIST_HEAD(&d40c
->queue
);
2836 INIT_LIST_HEAD(&d40c
->pending_queue
);
2837 INIT_LIST_HEAD(&d40c
->client
);
2838 INIT_LIST_HEAD(&d40c
->prepare_queue
);
2840 tasklet_init(&d40c
->tasklet
, dma_tasklet
,
2841 (unsigned long) d40c
);
2843 list_add_tail(&d40c
->chan
.device_node
,
2848 static void d40_ops_init(struct d40_base
*base
, struct dma_device
*dev
)
2850 if (dma_has_cap(DMA_SLAVE
, dev
->cap_mask
))
2851 dev
->device_prep_slave_sg
= d40_prep_slave_sg
;
2853 if (dma_has_cap(DMA_MEMCPY
, dev
->cap_mask
)) {
2854 dev
->device_prep_dma_memcpy
= d40_prep_memcpy
;
2857 * This controller can only access address at even
2858 * 32bit boundaries, i.e. 2^2
2860 dev
->copy_align
= 2;
2863 if (dma_has_cap(DMA_SG
, dev
->cap_mask
))
2864 dev
->device_prep_dma_sg
= d40_prep_memcpy_sg
;
2866 if (dma_has_cap(DMA_CYCLIC
, dev
->cap_mask
))
2867 dev
->device_prep_dma_cyclic
= dma40_prep_dma_cyclic
;
2869 dev
->device_alloc_chan_resources
= d40_alloc_chan_resources
;
2870 dev
->device_free_chan_resources
= d40_free_chan_resources
;
2871 dev
->device_issue_pending
= d40_issue_pending
;
2872 dev
->device_tx_status
= d40_tx_status
;
2873 dev
->device_control
= d40_control
;
2874 dev
->dev
= base
->dev
;
2877 static int __init
d40_dmaengine_init(struct d40_base
*base
,
2878 int num_reserved_chans
)
2882 d40_chan_init(base
, &base
->dma_slave
, base
->log_chans
,
2883 0, base
->num_log_chans
);
2885 dma_cap_zero(base
->dma_slave
.cap_mask
);
2886 dma_cap_set(DMA_SLAVE
, base
->dma_slave
.cap_mask
);
2887 dma_cap_set(DMA_CYCLIC
, base
->dma_slave
.cap_mask
);
2889 d40_ops_init(base
, &base
->dma_slave
);
2891 err
= dma_async_device_register(&base
->dma_slave
);
2894 d40_err(base
->dev
, "Failed to register slave channels\n");
2898 d40_chan_init(base
, &base
->dma_memcpy
, base
->log_chans
,
2899 base
->num_log_chans
, base
->num_memcpy_chans
);
2901 dma_cap_zero(base
->dma_memcpy
.cap_mask
);
2902 dma_cap_set(DMA_MEMCPY
, base
->dma_memcpy
.cap_mask
);
2903 dma_cap_set(DMA_SG
, base
->dma_memcpy
.cap_mask
);
2905 d40_ops_init(base
, &base
->dma_memcpy
);
2907 err
= dma_async_device_register(&base
->dma_memcpy
);
2911 "Failed to regsiter memcpy only channels\n");
2915 d40_chan_init(base
, &base
->dma_both
, base
->phy_chans
,
2916 0, num_reserved_chans
);
2918 dma_cap_zero(base
->dma_both
.cap_mask
);
2919 dma_cap_set(DMA_SLAVE
, base
->dma_both
.cap_mask
);
2920 dma_cap_set(DMA_MEMCPY
, base
->dma_both
.cap_mask
);
2921 dma_cap_set(DMA_SG
, base
->dma_both
.cap_mask
);
2922 dma_cap_set(DMA_CYCLIC
, base
->dma_slave
.cap_mask
);
2924 d40_ops_init(base
, &base
->dma_both
);
2925 err
= dma_async_device_register(&base
->dma_both
);
2929 "Failed to register logical and physical capable channels\n");
2934 dma_async_device_unregister(&base
->dma_memcpy
);
2936 dma_async_device_unregister(&base
->dma_slave
);
2941 /* Suspend resume functionality */
2942 #ifdef CONFIG_PM_SLEEP
2943 static int dma40_suspend(struct device
*dev
)
2945 struct platform_device
*pdev
= to_platform_device(dev
);
2946 struct d40_base
*base
= platform_get_drvdata(pdev
);
2949 ret
= pm_runtime_force_suspend(dev
);
2953 if (base
->lcpa_regulator
)
2954 ret
= regulator_disable(base
->lcpa_regulator
);
2958 static int dma40_resume(struct device
*dev
)
2960 struct platform_device
*pdev
= to_platform_device(dev
);
2961 struct d40_base
*base
= platform_get_drvdata(pdev
);
2964 if (base
->lcpa_regulator
) {
2965 ret
= regulator_enable(base
->lcpa_regulator
);
2970 return pm_runtime_force_resume(dev
);
2975 static void dma40_backup(void __iomem
*baseaddr
, u32
*backup
,
2976 u32
*regaddr
, int num
, bool save
)
2980 for (i
= 0; i
< num
; i
++) {
2981 void __iomem
*addr
= baseaddr
+ regaddr
[i
];
2984 backup
[i
] = readl_relaxed(addr
);
2986 writel_relaxed(backup
[i
], addr
);
2990 static void d40_save_restore_registers(struct d40_base
*base
, bool save
)
2994 /* Save/Restore channel specific registers */
2995 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
2999 if (base
->phy_res
[i
].reserved
)
3002 addr
= base
->virtbase
+ D40_DREG_PCBASE
+ i
* D40_DREG_PCDELTA
;
3003 idx
= i
* ARRAY_SIZE(d40_backup_regs_chan
);
3005 dma40_backup(addr
, &base
->reg_val_backup_chan
[idx
],
3006 d40_backup_regs_chan
,
3007 ARRAY_SIZE(d40_backup_regs_chan
),
3011 /* Save/Restore global registers */
3012 dma40_backup(base
->virtbase
, base
->reg_val_backup
,
3013 d40_backup_regs
, ARRAY_SIZE(d40_backup_regs
),
3016 /* Save/Restore registers only existing on dma40 v3 and later */
3017 if (base
->gen_dmac
.backup
)
3018 dma40_backup(base
->virtbase
, base
->reg_val_backup_v4
,
3019 base
->gen_dmac
.backup
,
3020 base
->gen_dmac
.backup_size
,
3024 static int dma40_runtime_suspend(struct device
*dev
)
3026 struct platform_device
*pdev
= to_platform_device(dev
);
3027 struct d40_base
*base
= platform_get_drvdata(pdev
);
3029 d40_save_restore_registers(base
, true);
3031 /* Don't disable/enable clocks for v1 due to HW bugs */
3033 writel_relaxed(base
->gcc_pwr_off_mask
,
3034 base
->virtbase
+ D40_DREG_GCC
);
3039 static int dma40_runtime_resume(struct device
*dev
)
3041 struct platform_device
*pdev
= to_platform_device(dev
);
3042 struct d40_base
*base
= platform_get_drvdata(pdev
);
3044 d40_save_restore_registers(base
, false);
3046 writel_relaxed(D40_DREG_GCC_ENABLE_ALL
,
3047 base
->virtbase
+ D40_DREG_GCC
);
3052 static const struct dev_pm_ops dma40_pm_ops
= {
3053 SET_LATE_SYSTEM_SLEEP_PM_OPS(dma40_suspend
, dma40_resume
)
3054 SET_PM_RUNTIME_PM_OPS(dma40_runtime_suspend
,
3055 dma40_runtime_resume
,
3059 /* Initialization functions. */
3061 static int __init
d40_phy_res_init(struct d40_base
*base
)
3064 int num_phy_chans_avail
= 0;
3066 int odd_even_bit
= -2;
3067 int gcc
= D40_DREG_GCC_ENA
;
3069 val
[0] = readl(base
->virtbase
+ D40_DREG_PRSME
);
3070 val
[1] = readl(base
->virtbase
+ D40_DREG_PRSMO
);
3072 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
3073 base
->phy_res
[i
].num
= i
;
3074 odd_even_bit
+= 2 * ((i
% 2) == 0);
3075 if (((val
[i
% 2] >> odd_even_bit
) & 3) == 1) {
3076 /* Mark security only channels as occupied */
3077 base
->phy_res
[i
].allocated_src
= D40_ALLOC_PHY
;
3078 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_PHY
;
3079 base
->phy_res
[i
].reserved
= true;
3080 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i
),
3082 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(i
),
3087 base
->phy_res
[i
].allocated_src
= D40_ALLOC_FREE
;
3088 base
->phy_res
[i
].allocated_dst
= D40_ALLOC_FREE
;
3089 base
->phy_res
[i
].reserved
= false;
3090 num_phy_chans_avail
++;
3092 spin_lock_init(&base
->phy_res
[i
].lock
);
3095 /* Mark disabled channels as occupied */
3096 for (i
= 0; base
->plat_data
->disabled_channels
[i
] != -1; i
++) {
3097 int chan
= base
->plat_data
->disabled_channels
[i
];
3099 base
->phy_res
[chan
].allocated_src
= D40_ALLOC_PHY
;
3100 base
->phy_res
[chan
].allocated_dst
= D40_ALLOC_PHY
;
3101 base
->phy_res
[chan
].reserved
= true;
3102 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan
),
3104 gcc
|= D40_DREG_GCC_EVTGRP_ENA(D40_PHYS_TO_GROUP(chan
),
3106 num_phy_chans_avail
--;
3109 /* Mark soft_lli channels */
3110 for (i
= 0; i
< base
->plat_data
->num_of_soft_lli_chans
; i
++) {
3111 int chan
= base
->plat_data
->soft_lli_chans
[i
];
3113 base
->phy_res
[chan
].use_soft_lli
= true;
3116 dev_info(base
->dev
, "%d of %d physical DMA channels available\n",
3117 num_phy_chans_avail
, base
->num_phy_chans
);
3119 /* Verify settings extended vs standard */
3120 val
[0] = readl(base
->virtbase
+ D40_DREG_PRTYP
);
3122 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
3124 if (base
->phy_res
[i
].allocated_src
== D40_ALLOC_FREE
&&
3125 (val
[0] & 0x3) != 1)
3127 "[%s] INFO: channel %d is misconfigured (%d)\n",
3128 __func__
, i
, val
[0] & 0x3);
3130 val
[0] = val
[0] >> 2;
3134 * To keep things simple, Enable all clocks initially.
3135 * The clocks will get managed later post channel allocation.
3136 * The clocks for the event lines on which reserved channels exists
3137 * are not managed here.
3139 writel(D40_DREG_GCC_ENABLE_ALL
, base
->virtbase
+ D40_DREG_GCC
);
3140 base
->gcc_pwr_off_mask
= gcc
;
3142 return num_phy_chans_avail
;
3145 static struct d40_base
* __init
d40_hw_detect_init(struct platform_device
*pdev
)
3147 struct stedma40_platform_data
*plat_data
= dev_get_platdata(&pdev
->dev
);
3148 struct clk
*clk
= NULL
;
3149 void __iomem
*virtbase
= NULL
;
3150 struct resource
*res
= NULL
;
3151 struct d40_base
*base
= NULL
;
3152 int num_log_chans
= 0;
3154 int num_memcpy_chans
;
3155 int clk_ret
= -EINVAL
;
3161 clk
= clk_get(&pdev
->dev
, NULL
);
3163 d40_err(&pdev
->dev
, "No matching clock found\n");
3167 clk_ret
= clk_prepare_enable(clk
);
3169 d40_err(&pdev
->dev
, "Failed to prepare/enable clock\n");
3173 /* Get IO for DMAC base address */
3174 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "base");
3178 if (request_mem_region(res
->start
, resource_size(res
),
3179 D40_NAME
" I/O base") == NULL
)
3182 virtbase
= ioremap(res
->start
, resource_size(res
));
3186 /* This is just a regular AMBA PrimeCell ID actually */
3187 for (pid
= 0, i
= 0; i
< 4; i
++)
3188 pid
|= (readl(virtbase
+ resource_size(res
) - 0x20 + 4 * i
)
3190 for (cid
= 0, i
= 0; i
< 4; i
++)
3191 cid
|= (readl(virtbase
+ resource_size(res
) - 0x10 + 4 * i
)
3194 if (cid
!= AMBA_CID
) {
3195 d40_err(&pdev
->dev
, "Unknown hardware! No PrimeCell ID\n");
3198 if (AMBA_MANF_BITS(pid
) != AMBA_VENDOR_ST
) {
3199 d40_err(&pdev
->dev
, "Unknown designer! Got %x wanted %x\n",
3200 AMBA_MANF_BITS(pid
),
3206 * DB8500ed has revision 0
3208 * DB8500v1 has revision 2
3209 * DB8500v2 has revision 3
3210 * AP9540v1 has revision 4
3211 * DB8540v1 has revision 4
3213 rev
= AMBA_REV_BITS(pid
);
3215 d40_err(&pdev
->dev
, "hardware revision: %d is not supported", rev
);
3219 /* The number of physical channels on this HW */
3220 if (plat_data
->num_of_phy_chans
)
3221 num_phy_chans
= plat_data
->num_of_phy_chans
;
3223 num_phy_chans
= 4 * (readl(virtbase
+ D40_DREG_ICFG
) & 0x7) + 4;
3225 /* The number of channels used for memcpy */
3226 if (plat_data
->num_of_memcpy_chans
)
3227 num_memcpy_chans
= plat_data
->num_of_memcpy_chans
;
3229 num_memcpy_chans
= ARRAY_SIZE(dma40_memcpy_channels
);
3231 num_log_chans
= num_phy_chans
* D40_MAX_LOG_CHAN_PER_PHY
;
3233 dev_info(&pdev
->dev
,
3234 "hardware rev: %d @ %pa with %d physical and %d logical channels\n",
3235 rev
, &res
->start
, num_phy_chans
, num_log_chans
);
3237 base
= kzalloc(ALIGN(sizeof(struct d40_base
), 4) +
3238 (num_phy_chans
+ num_log_chans
+ num_memcpy_chans
) *
3239 sizeof(struct d40_chan
), GFP_KERNEL
);
3242 d40_err(&pdev
->dev
, "Out of memory\n");
3248 base
->num_memcpy_chans
= num_memcpy_chans
;
3249 base
->num_phy_chans
= num_phy_chans
;
3250 base
->num_log_chans
= num_log_chans
;
3251 base
->phy_start
= res
->start
;
3252 base
->phy_size
= resource_size(res
);
3253 base
->virtbase
= virtbase
;
3254 base
->plat_data
= plat_data
;
3255 base
->dev
= &pdev
->dev
;
3256 base
->phy_chans
= ((void *)base
) + ALIGN(sizeof(struct d40_base
), 4);
3257 base
->log_chans
= &base
->phy_chans
[num_phy_chans
];
3259 if (base
->plat_data
->num_of_phy_chans
== 14) {
3260 base
->gen_dmac
.backup
= d40_backup_regs_v4b
;
3261 base
->gen_dmac
.backup_size
= BACKUP_REGS_SZ_V4B
;
3262 base
->gen_dmac
.interrupt_en
= D40_DREG_CPCMIS
;
3263 base
->gen_dmac
.interrupt_clear
= D40_DREG_CPCICR
;
3264 base
->gen_dmac
.realtime_en
= D40_DREG_CRSEG1
;
3265 base
->gen_dmac
.realtime_clear
= D40_DREG_CRCEG1
;
3266 base
->gen_dmac
.high_prio_en
= D40_DREG_CPSEG1
;
3267 base
->gen_dmac
.high_prio_clear
= D40_DREG_CPCEG1
;
3268 base
->gen_dmac
.il
= il_v4b
;
3269 base
->gen_dmac
.il_size
= ARRAY_SIZE(il_v4b
);
3270 base
->gen_dmac
.init_reg
= dma_init_reg_v4b
;
3271 base
->gen_dmac
.init_reg_size
= ARRAY_SIZE(dma_init_reg_v4b
);
3273 if (base
->rev
>= 3) {
3274 base
->gen_dmac
.backup
= d40_backup_regs_v4a
;
3275 base
->gen_dmac
.backup_size
= BACKUP_REGS_SZ_V4A
;
3277 base
->gen_dmac
.interrupt_en
= D40_DREG_PCMIS
;
3278 base
->gen_dmac
.interrupt_clear
= D40_DREG_PCICR
;
3279 base
->gen_dmac
.realtime_en
= D40_DREG_RSEG1
;
3280 base
->gen_dmac
.realtime_clear
= D40_DREG_RCEG1
;
3281 base
->gen_dmac
.high_prio_en
= D40_DREG_PSEG1
;
3282 base
->gen_dmac
.high_prio_clear
= D40_DREG_PCEG1
;
3283 base
->gen_dmac
.il
= il_v4a
;
3284 base
->gen_dmac
.il_size
= ARRAY_SIZE(il_v4a
);
3285 base
->gen_dmac
.init_reg
= dma_init_reg_v4a
;
3286 base
->gen_dmac
.init_reg_size
= ARRAY_SIZE(dma_init_reg_v4a
);
3289 base
->phy_res
= kzalloc(num_phy_chans
* sizeof(struct d40_phy_res
),
3294 base
->lookup_phy_chans
= kzalloc(num_phy_chans
*
3295 sizeof(struct d40_chan
*),
3297 if (!base
->lookup_phy_chans
)
3300 base
->lookup_log_chans
= kzalloc(num_log_chans
*
3301 sizeof(struct d40_chan
*),
3303 if (!base
->lookup_log_chans
)
3306 base
->reg_val_backup_chan
= kmalloc(base
->num_phy_chans
*
3307 sizeof(d40_backup_regs_chan
),
3309 if (!base
->reg_val_backup_chan
)
3312 base
->lcla_pool
.alloc_map
=
3313 kzalloc(num_phy_chans
* sizeof(struct d40_desc
*)
3314 * D40_LCLA_LINK_PER_EVENT_GRP
, GFP_KERNEL
);
3315 if (!base
->lcla_pool
.alloc_map
)
3318 base
->desc_slab
= kmem_cache_create(D40_NAME
, sizeof(struct d40_desc
),
3319 0, SLAB_HWCACHE_ALIGN
,
3321 if (base
->desc_slab
== NULL
)
3328 clk_disable_unprepare(clk
);
3334 release_mem_region(res
->start
,
3335 resource_size(res
));
3340 kfree(base
->lcla_pool
.alloc_map
);
3341 kfree(base
->reg_val_backup_chan
);
3342 kfree(base
->lookup_log_chans
);
3343 kfree(base
->lookup_phy_chans
);
3344 kfree(base
->phy_res
);
3351 static void __init
d40_hw_init(struct d40_base
*base
)
3355 u32 prmseo
[2] = {0, 0};
3356 u32 activeo
[2] = {0xFFFFFFFF, 0xFFFFFFFF};
3359 struct d40_reg_val
*dma_init_reg
= base
->gen_dmac
.init_reg
;
3360 u32 reg_size
= base
->gen_dmac
.init_reg_size
;
3362 for (i
= 0; i
< reg_size
; i
++)
3363 writel(dma_init_reg
[i
].val
,
3364 base
->virtbase
+ dma_init_reg
[i
].reg
);
3366 /* Configure all our dma channels to default settings */
3367 for (i
= 0; i
< base
->num_phy_chans
; i
++) {
3369 activeo
[i
% 2] = activeo
[i
% 2] << 2;
3371 if (base
->phy_res
[base
->num_phy_chans
- i
- 1].allocated_src
3373 activeo
[i
% 2] |= 3;
3377 /* Enable interrupt # */
3378 pcmis
= (pcmis
<< 1) | 1;
3380 /* Clear interrupt # */
3381 pcicr
= (pcicr
<< 1) | 1;
3383 /* Set channel to physical mode */
3384 prmseo
[i
% 2] = prmseo
[i
% 2] << 2;
3389 writel(prmseo
[1], base
->virtbase
+ D40_DREG_PRMSE
);
3390 writel(prmseo
[0], base
->virtbase
+ D40_DREG_PRMSO
);
3391 writel(activeo
[1], base
->virtbase
+ D40_DREG_ACTIVE
);
3392 writel(activeo
[0], base
->virtbase
+ D40_DREG_ACTIVO
);
3394 /* Write which interrupt to enable */
3395 writel(pcmis
, base
->virtbase
+ base
->gen_dmac
.interrupt_en
);
3397 /* Write which interrupt to clear */
3398 writel(pcicr
, base
->virtbase
+ base
->gen_dmac
.interrupt_clear
);
3400 /* These are __initdata and cannot be accessed after init */
3401 base
->gen_dmac
.init_reg
= NULL
;
3402 base
->gen_dmac
.init_reg_size
= 0;
3405 static int __init
d40_lcla_allocate(struct d40_base
*base
)
3407 struct d40_lcla_pool
*pool
= &base
->lcla_pool
;
3408 unsigned long *page_list
;
3413 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
3414 * To full fill this hardware requirement without wasting 256 kb
3415 * we allocate pages until we get an aligned one.
3417 page_list
= kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS
,
3425 /* Calculating how many pages that are required */
3426 base
->lcla_pool
.pages
= SZ_1K
* base
->num_phy_chans
/ PAGE_SIZE
;
3428 for (i
= 0; i
< MAX_LCLA_ALLOC_ATTEMPTS
; i
++) {
3429 page_list
[i
] = __get_free_pages(GFP_KERNEL
,
3430 base
->lcla_pool
.pages
);
3431 if (!page_list
[i
]) {
3433 d40_err(base
->dev
, "Failed to allocate %d pages.\n",
3434 base
->lcla_pool
.pages
);
3436 for (j
= 0; j
< i
; j
++)
3437 free_pages(page_list
[j
], base
->lcla_pool
.pages
);
3441 if ((virt_to_phys((void *)page_list
[i
]) &
3442 (LCLA_ALIGNMENT
- 1)) == 0)
3446 for (j
= 0; j
< i
; j
++)
3447 free_pages(page_list
[j
], base
->lcla_pool
.pages
);
3449 if (i
< MAX_LCLA_ALLOC_ATTEMPTS
) {
3450 base
->lcla_pool
.base
= (void *)page_list
[i
];
3453 * After many attempts and no succees with finding the correct
3454 * alignment, try with allocating a big buffer.
3457 "[%s] Failed to get %d pages @ 18 bit align.\n",
3458 __func__
, base
->lcla_pool
.pages
);
3459 base
->lcla_pool
.base_unaligned
= kmalloc(SZ_1K
*
3460 base
->num_phy_chans
+
3463 if (!base
->lcla_pool
.base_unaligned
) {
3468 base
->lcla_pool
.base
= PTR_ALIGN(base
->lcla_pool
.base_unaligned
,
3472 pool
->dma_addr
= dma_map_single(base
->dev
, pool
->base
,
3473 SZ_1K
* base
->num_phy_chans
,
3475 if (dma_mapping_error(base
->dev
, pool
->dma_addr
)) {
3481 writel(virt_to_phys(base
->lcla_pool
.base
),
3482 base
->virtbase
+ D40_DREG_LCLA
);
3488 static int __init
d40_of_probe(struct platform_device
*pdev
,
3489 struct device_node
*np
)
3491 struct stedma40_platform_data
*pdata
;
3492 int num_phy
= 0, num_memcpy
= 0, num_disabled
= 0;
3495 pdata
= devm_kzalloc(&pdev
->dev
,
3496 sizeof(struct stedma40_platform_data
),
3501 /* If absent this value will be obtained from h/w. */
3502 of_property_read_u32(np
, "dma-channels", &num_phy
);
3504 pdata
->num_of_phy_chans
= num_phy
;
3506 list
= of_get_property(np
, "memcpy-channels", &num_memcpy
);
3507 num_memcpy
/= sizeof(*list
);
3509 if (num_memcpy
> D40_MEMCPY_MAX_CHANS
|| num_memcpy
<= 0) {
3511 "Invalid number of memcpy channels specified (%d)\n",
3515 pdata
->num_of_memcpy_chans
= num_memcpy
;
3517 of_property_read_u32_array(np
, "memcpy-channels",
3518 dma40_memcpy_channels
,
3521 list
= of_get_property(np
, "disabled-channels", &num_disabled
);
3522 num_disabled
/= sizeof(*list
);
3524 if (num_disabled
>= STEDMA40_MAX_PHYS
|| num_disabled
< 0) {
3526 "Invalid number of disabled channels specified (%d)\n",
3531 of_property_read_u32_array(np
, "disabled-channels",
3532 pdata
->disabled_channels
,
3534 pdata
->disabled_channels
[num_disabled
] = -1;
3536 pdev
->dev
.platform_data
= pdata
;
3541 static int __init
d40_probe(struct platform_device
*pdev
)
3543 struct stedma40_platform_data
*plat_data
= dev_get_platdata(&pdev
->dev
);
3544 struct device_node
*np
= pdev
->dev
.of_node
;
3546 struct d40_base
*base
= NULL
;
3547 struct resource
*res
= NULL
;
3548 int num_reserved_chans
;
3553 if(d40_of_probe(pdev
, np
)) {
3558 d40_err(&pdev
->dev
, "No pdata or Device Tree provided\n");
3563 base
= d40_hw_detect_init(pdev
);
3567 num_reserved_chans
= d40_phy_res_init(base
);
3569 platform_set_drvdata(pdev
, base
);
3571 spin_lock_init(&base
->interrupt_lock
);
3572 spin_lock_init(&base
->execmd_lock
);
3574 /* Get IO for logical channel parameter address */
3575 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
, "lcpa");
3578 d40_err(&pdev
->dev
, "No \"lcpa\" memory resource\n");
3581 base
->lcpa_size
= resource_size(res
);
3582 base
->phy_lcpa
= res
->start
;
3584 if (request_mem_region(res
->start
, resource_size(res
),
3585 D40_NAME
" I/O lcpa") == NULL
) {
3587 d40_err(&pdev
->dev
, "Failed to request LCPA region %pR\n", res
);
3591 /* We make use of ESRAM memory for this. */
3592 val
= readl(base
->virtbase
+ D40_DREG_LCPA
);
3593 if (res
->start
!= val
&& val
!= 0) {
3594 dev_warn(&pdev
->dev
,
3595 "[%s] Mismatch LCPA dma 0x%x, def %pa\n",
3596 __func__
, val
, &res
->start
);
3598 writel(res
->start
, base
->virtbase
+ D40_DREG_LCPA
);
3600 base
->lcpa_base
= ioremap(res
->start
, resource_size(res
));
3601 if (!base
->lcpa_base
) {
3603 d40_err(&pdev
->dev
, "Failed to ioremap LCPA region\n");
3606 /* If lcla has to be located in ESRAM we don't need to allocate */
3607 if (base
->plat_data
->use_esram_lcla
) {
3608 res
= platform_get_resource_byname(pdev
, IORESOURCE_MEM
,
3613 "No \"lcla_esram\" memory resource\n");
3616 base
->lcla_pool
.base
= ioremap(res
->start
,
3617 resource_size(res
));
3618 if (!base
->lcla_pool
.base
) {
3620 d40_err(&pdev
->dev
, "Failed to ioremap LCLA region\n");
3623 writel(res
->start
, base
->virtbase
+ D40_DREG_LCLA
);
3626 ret
= d40_lcla_allocate(base
);
3628 d40_err(&pdev
->dev
, "Failed to allocate LCLA area\n");
3633 spin_lock_init(&base
->lcla_pool
.lock
);
3635 base
->irq
= platform_get_irq(pdev
, 0);
3637 ret
= request_irq(base
->irq
, d40_handle_interrupt
, 0, D40_NAME
, base
);
3639 d40_err(&pdev
->dev
, "No IRQ defined\n");
3643 if (base
->plat_data
->use_esram_lcla
) {
3645 base
->lcpa_regulator
= regulator_get(base
->dev
, "lcla_esram");
3646 if (IS_ERR(base
->lcpa_regulator
)) {
3647 d40_err(&pdev
->dev
, "Failed to get lcpa_regulator\n");
3648 ret
= PTR_ERR(base
->lcpa_regulator
);
3649 base
->lcpa_regulator
= NULL
;
3653 ret
= regulator_enable(base
->lcpa_regulator
);
3656 "Failed to enable lcpa_regulator\n");
3657 regulator_put(base
->lcpa_regulator
);
3658 base
->lcpa_regulator
= NULL
;
3663 writel_relaxed(D40_DREG_GCC_ENABLE_ALL
, base
->virtbase
+ D40_DREG_GCC
);
3665 pm_runtime_irq_safe(base
->dev
);
3666 pm_runtime_set_autosuspend_delay(base
->dev
, DMA40_AUTOSUSPEND_DELAY
);
3667 pm_runtime_use_autosuspend(base
->dev
);
3668 pm_runtime_mark_last_busy(base
->dev
);
3669 pm_runtime_set_active(base
->dev
);
3670 pm_runtime_enable(base
->dev
);
3672 ret
= d40_dmaengine_init(base
, num_reserved_chans
);
3676 base
->dev
->dma_parms
= &base
->dma_parms
;
3677 ret
= dma_set_max_seg_size(base
->dev
, STEDMA40_MAX_SEG_SIZE
);
3679 d40_err(&pdev
->dev
, "Failed to set dma max seg size\n");
3686 ret
= of_dma_controller_register(np
, d40_xlate
, NULL
);
3689 "could not register of_dma_controller\n");
3692 dev_info(base
->dev
, "initialized\n");
3697 if (base
->desc_slab
)
3698 kmem_cache_destroy(base
->desc_slab
);
3700 iounmap(base
->virtbase
);
3702 if (base
->lcla_pool
.base
&& base
->plat_data
->use_esram_lcla
) {
3703 iounmap(base
->lcla_pool
.base
);
3704 base
->lcla_pool
.base
= NULL
;
3707 if (base
->lcla_pool
.dma_addr
)
3708 dma_unmap_single(base
->dev
, base
->lcla_pool
.dma_addr
,
3709 SZ_1K
* base
->num_phy_chans
,
3712 if (!base
->lcla_pool
.base_unaligned
&& base
->lcla_pool
.base
)
3713 free_pages((unsigned long)base
->lcla_pool
.base
,
3714 base
->lcla_pool
.pages
);
3716 kfree(base
->lcla_pool
.base_unaligned
);
3719 release_mem_region(base
->phy_lcpa
,
3721 if (base
->phy_start
)
3722 release_mem_region(base
->phy_start
,
3725 clk_disable_unprepare(base
->clk
);
3729 if (base
->lcpa_regulator
) {
3730 regulator_disable(base
->lcpa_regulator
);
3731 regulator_put(base
->lcpa_regulator
);
3734 kfree(base
->lcla_pool
.alloc_map
);
3735 kfree(base
->lookup_log_chans
);
3736 kfree(base
->lookup_phy_chans
);
3737 kfree(base
->phy_res
);
3741 d40_err(&pdev
->dev
, "probe failed\n");
3745 static const struct of_device_id d40_match
[] = {
3746 { .compatible
= "stericsson,dma40", },
3750 static struct platform_driver d40_driver
= {
3752 .owner
= THIS_MODULE
,
3754 .pm
= &dma40_pm_ops
,
3755 .of_match_table
= d40_match
,
3759 static int __init
stedma40_init(void)
3761 return platform_driver_probe(&d40_driver
, d40_probe
);
3763 subsys_initcall(stedma40_init
);