3 * Guennadi Liakhovetski, DENX Software Engineering, <lg@denx.de>
5 * Copyright (C) 2005-2007 Freescale Semiconductor, Inc. All Rights Reserved.
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/dma-mapping.h>
13 #include <linux/init.h>
14 #include <linux/platform_device.h>
15 #include <linux/err.h>
16 #include <linux/spinlock.h>
17 #include <linux/delay.h>
18 #include <linux/list.h>
19 #include <linux/clk.h>
20 #include <linux/vmalloc.h>
21 #include <linux/string.h>
22 #include <linux/interrupt.h>
24 #include <linux/module.h>
28 #include "ipu_intern.h"
30 #define FS_VF_IN_VALID 0x00000002
31 #define FS_ENC_IN_VALID 0x00000001
33 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
37 * There can be only one, we could allocate it dynamically, but then we'd have
38 * to add an extra parameter to some functions, and use something as ugly as
39 * struct ipu *ipu = to_ipu(to_idmac(ichan->dma_chan.device));
42 static struct ipu ipu_data
;
44 #define to_ipu(id) container_of(id, struct ipu, idmac)
46 static u32
__idmac_read_icreg(struct ipu
*ipu
, unsigned long reg
)
48 return __raw_readl(ipu
->reg_ic
+ reg
);
51 #define idmac_read_icreg(ipu, reg) __idmac_read_icreg(ipu, reg - IC_CONF)
53 static void __idmac_write_icreg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
55 __raw_writel(value
, ipu
->reg_ic
+ reg
);
58 #define idmac_write_icreg(ipu, v, reg) __idmac_write_icreg(ipu, v, reg - IC_CONF)
60 static u32
idmac_read_ipureg(struct ipu
*ipu
, unsigned long reg
)
62 return __raw_readl(ipu
->reg_ipu
+ reg
);
65 static void idmac_write_ipureg(struct ipu
*ipu
, u32 value
, unsigned long reg
)
67 __raw_writel(value
, ipu
->reg_ipu
+ reg
);
70 /*****************************************************************************
71 * IPU / IC common functions
73 static void dump_idmac_reg(struct ipu
*ipu
)
75 dev_dbg(ipu
->dev
, "IDMAC_CONF 0x%x, IC_CONF 0x%x, IDMAC_CHA_EN 0x%x, "
76 "IDMAC_CHA_PRI 0x%x, IDMAC_CHA_BUSY 0x%x\n",
77 idmac_read_icreg(ipu
, IDMAC_CONF
),
78 idmac_read_icreg(ipu
, IC_CONF
),
79 idmac_read_icreg(ipu
, IDMAC_CHA_EN
),
80 idmac_read_icreg(ipu
, IDMAC_CHA_PRI
),
81 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
));
82 dev_dbg(ipu
->dev
, "BUF0_RDY 0x%x, BUF1_RDY 0x%x, CUR_BUF 0x%x, "
83 "DB_MODE 0x%x, TASKS_STAT 0x%x\n",
84 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
85 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
86 idmac_read_ipureg(ipu
, IPU_CHA_CUR_BUF
),
87 idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
),
88 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
91 static uint32_t bytes_per_pixel(enum pixel_fmt fmt
)
94 case IPU_PIX_FMT_GENERIC
: /* generic data */
95 case IPU_PIX_FMT_RGB332
:
96 case IPU_PIX_FMT_YUV420P
:
97 case IPU_PIX_FMT_YUV422P
:
100 case IPU_PIX_FMT_RGB565
:
101 case IPU_PIX_FMT_YUYV
:
102 case IPU_PIX_FMT_UYVY
:
104 case IPU_PIX_FMT_BGR24
:
105 case IPU_PIX_FMT_RGB24
:
107 case IPU_PIX_FMT_GENERIC_32
: /* generic data */
108 case IPU_PIX_FMT_BGR32
:
109 case IPU_PIX_FMT_RGB32
:
110 case IPU_PIX_FMT_ABGR32
:
115 /* Enable direct write to memory by the Camera Sensor Interface */
116 static void ipu_ic_enable_task(struct ipu
*ipu
, enum ipu_channel channel
)
118 uint32_t ic_conf
, mask
;
122 mask
= IC_CONF_PRPENC_EN
;
125 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
130 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) | mask
;
131 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
134 /* Called under spin_lock_irqsave(&ipu_data.lock) */
135 static void ipu_ic_disable_task(struct ipu
*ipu
, enum ipu_channel channel
)
137 uint32_t ic_conf
, mask
;
141 mask
= IC_CONF_PRPENC_EN
;
144 mask
= IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
;
149 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
) & ~mask
;
150 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
153 static uint32_t ipu_channel_status(struct ipu
*ipu
, enum ipu_channel channel
)
155 uint32_t stat
= TASK_STAT_IDLE
;
156 uint32_t task_stat_reg
= idmac_read_ipureg(ipu
, IPU_TASKS_STAT
);
160 stat
= (task_stat_reg
& TSTAT_CSI2MEM_MASK
) >>
161 TSTAT_CSI2MEM_OFFSET
;
172 struct chan_param_mem_planar
{
210 } __attribute__ ((packed
));
212 struct chan_param_mem_interleaved
{
269 } __attribute__ ((packed
));
271 union chan_param_mem
{
272 struct chan_param_mem_planar pp
;
273 struct chan_param_mem_interleaved ip
;
276 static void ipu_ch_param_set_plane_offset(union chan_param_mem
*params
,
277 u32 u_offset
, u32 v_offset
)
279 params
->pp
.ubo_l
= u_offset
& 0x7ff;
280 params
->pp
.ubo_h
= u_offset
>> 11;
281 params
->pp
.vbo_l
= v_offset
& 0x1ffff;
282 params
->pp
.vbo_h
= v_offset
>> 17;
285 static void ipu_ch_param_set_size(union chan_param_mem
*params
,
286 uint32_t pixel_fmt
, uint16_t width
,
287 uint16_t height
, uint16_t stride
)
292 params
->pp
.fw
= width
- 1;
293 params
->pp
.fh_l
= height
- 1;
294 params
->pp
.fh_h
= (height
- 1) >> 8;
295 params
->pp
.sl
= stride
- 1;
298 case IPU_PIX_FMT_GENERIC
:
299 /*Represents 8-bit Generic data */
303 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
305 case IPU_PIX_FMT_GENERIC_32
:
306 /*Represents 32-bit Generic data */
310 params
->pp
.sat
= 2; /* SAT = use 32-bit access */
312 case IPU_PIX_FMT_RGB565
:
316 params
->ip
.sat
= 2; /* SAT = 32-bit access */
317 params
->ip
.ofs0
= 0; /* Red bit offset */
318 params
->ip
.ofs1
= 5; /* Green bit offset */
319 params
->ip
.ofs2
= 11; /* Blue bit offset */
320 params
->ip
.ofs3
= 16; /* Alpha bit offset */
321 params
->ip
.wid0
= 4; /* Red bit width - 1 */
322 params
->ip
.wid1
= 5; /* Green bit width - 1 */
323 params
->ip
.wid2
= 4; /* Blue bit width - 1 */
325 case IPU_PIX_FMT_BGR24
:
326 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
329 params
->ip
.sat
= 2; /* SAT = 32-bit access */
330 params
->ip
.ofs0
= 0; /* Red bit offset */
331 params
->ip
.ofs1
= 8; /* Green bit offset */
332 params
->ip
.ofs2
= 16; /* Blue bit offset */
333 params
->ip
.ofs3
= 24; /* Alpha bit offset */
334 params
->ip
.wid0
= 7; /* Red bit width - 1 */
335 params
->ip
.wid1
= 7; /* Green bit width - 1 */
336 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
338 case IPU_PIX_FMT_RGB24
:
339 params
->ip
.bpp
= 1; /* 24 BPP & RGB PFS */
342 params
->ip
.sat
= 2; /* SAT = 32-bit access */
343 params
->ip
.ofs0
= 16; /* Red bit offset */
344 params
->ip
.ofs1
= 8; /* Green bit offset */
345 params
->ip
.ofs2
= 0; /* Blue bit offset */
346 params
->ip
.ofs3
= 24; /* Alpha bit offset */
347 params
->ip
.wid0
= 7; /* Red bit width - 1 */
348 params
->ip
.wid1
= 7; /* Green bit width - 1 */
349 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
351 case IPU_PIX_FMT_BGRA32
:
352 case IPU_PIX_FMT_BGR32
:
353 case IPU_PIX_FMT_ABGR32
:
357 params
->ip
.sat
= 2; /* SAT = 32-bit access */
358 params
->ip
.ofs0
= 8; /* Red bit offset */
359 params
->ip
.ofs1
= 16; /* Green bit offset */
360 params
->ip
.ofs2
= 24; /* Blue bit offset */
361 params
->ip
.ofs3
= 0; /* Alpha bit offset */
362 params
->ip
.wid0
= 7; /* Red bit width - 1 */
363 params
->ip
.wid1
= 7; /* Green bit width - 1 */
364 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
365 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
367 case IPU_PIX_FMT_RGBA32
:
368 case IPU_PIX_FMT_RGB32
:
372 params
->ip
.sat
= 2; /* SAT = 32-bit access */
373 params
->ip
.ofs0
= 24; /* Red bit offset */
374 params
->ip
.ofs1
= 16; /* Green bit offset */
375 params
->ip
.ofs2
= 8; /* Blue bit offset */
376 params
->ip
.ofs3
= 0; /* Alpha bit offset */
377 params
->ip
.wid0
= 7; /* Red bit width - 1 */
378 params
->ip
.wid1
= 7; /* Green bit width - 1 */
379 params
->ip
.wid2
= 7; /* Blue bit width - 1 */
380 params
->ip
.wid3
= 7; /* Alpha bit width - 1 */
382 case IPU_PIX_FMT_UYVY
:
386 params
->ip
.sat
= 2; /* SAT = 32-bit access */
388 case IPU_PIX_FMT_YUV420P2
:
389 case IPU_PIX_FMT_YUV420P
:
393 params
->ip
.sat
= 2; /* SAT = 32-bit access */
394 u_offset
= stride
* height
;
395 v_offset
= u_offset
+ u_offset
/ 4;
396 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
398 case IPU_PIX_FMT_YVU422P
:
402 params
->ip
.sat
= 2; /* SAT = 32-bit access */
403 v_offset
= stride
* height
;
404 u_offset
= v_offset
+ v_offset
/ 2;
405 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
407 case IPU_PIX_FMT_YUV422P
:
411 params
->ip
.sat
= 2; /* SAT = 32-bit access */
412 u_offset
= stride
* height
;
413 v_offset
= u_offset
+ u_offset
/ 2;
414 ipu_ch_param_set_plane_offset(params
, u_offset
, v_offset
);
417 dev_err(ipu_data
.dev
,
418 "mx3 ipu: unimplemented pixel format %d\n", pixel_fmt
);
425 static void ipu_ch_param_set_burst_size(union chan_param_mem
*params
,
426 uint16_t burst_pixels
)
428 params
->pp
.npb
= burst_pixels
- 1;
431 static void ipu_ch_param_set_buffer(union chan_param_mem
*params
,
432 dma_addr_t buf0
, dma_addr_t buf1
)
434 params
->pp
.eba0
= buf0
;
435 params
->pp
.eba1
= buf1
;
438 static void ipu_ch_param_set_rotation(union chan_param_mem
*params
,
439 enum ipu_rotate_mode rotate
)
441 params
->pp
.bam
= rotate
;
444 static void ipu_write_param_mem(uint32_t addr
, uint32_t *data
,
447 for (; num_words
> 0; num_words
--) {
448 dev_dbg(ipu_data
.dev
,
449 "write param mem - addr = 0x%08X, data = 0x%08X\n",
451 idmac_write_ipureg(&ipu_data
, addr
, IPU_IMA_ADDR
);
452 idmac_write_ipureg(&ipu_data
, *data
++, IPU_IMA_DATA
);
454 if ((addr
& 0x7) == 5) {
455 addr
&= ~0x7; /* set to word 0 */
456 addr
+= 8; /* increment to next row */
461 static int calc_resize_coeffs(uint32_t in_size
, uint32_t out_size
,
462 uint32_t *resize_coeff
,
463 uint32_t *downsize_coeff
)
466 uint32_t temp_downsize
;
468 *resize_coeff
= 1 << 13;
469 *downsize_coeff
= 1 << 13;
471 /* Cannot downsize more than 8:1 */
472 if (out_size
<< 3 < in_size
)
475 /* compute downsizing coefficient */
478 while (temp_size
>= out_size
* 2 && temp_downsize
< 2) {
482 *downsize_coeff
= temp_downsize
;
485 * compute resizing coefficient using the following formula:
486 * resize_coeff = M*(SI -1)/(SO - 1)
487 * where M = 2^13, SI - input size, SO - output size
489 *resize_coeff
= (8192L * (temp_size
- 1)) / (out_size
- 1);
490 if (*resize_coeff
>= 16384L) {
491 dev_err(ipu_data
.dev
, "Warning! Overflow on resize coeff.\n");
492 *resize_coeff
= 0x3FFF;
495 dev_dbg(ipu_data
.dev
, "resizing from %u -> %u pixels, "
496 "downsize=%u, resize=%u.%lu (reg=%u)\n", in_size
, out_size
,
497 *downsize_coeff
, *resize_coeff
>= 8192L ? 1 : 0,
498 ((*resize_coeff
& 0x1FFF) * 10000L) / 8192L, *resize_coeff
);
503 static enum ipu_color_space
format_to_colorspace(enum pixel_fmt fmt
)
506 case IPU_PIX_FMT_RGB565
:
507 case IPU_PIX_FMT_BGR24
:
508 case IPU_PIX_FMT_RGB24
:
509 case IPU_PIX_FMT_BGR32
:
510 case IPU_PIX_FMT_RGB32
:
511 return IPU_COLORSPACE_RGB
;
513 return IPU_COLORSPACE_YCBCR
;
517 static int ipu_ic_init_prpenc(struct ipu
*ipu
,
518 union ipu_channel_param
*params
, bool src_is_csi
)
520 uint32_t reg
, ic_conf
;
521 uint32_t downsize_coeff
, resize_coeff
;
522 enum ipu_color_space in_fmt
, out_fmt
;
524 /* Setup vertical resizing */
525 calc_resize_coeffs(params
->video
.in_height
,
526 params
->video
.out_height
,
527 &resize_coeff
, &downsize_coeff
);
528 reg
= (downsize_coeff
<< 30) | (resize_coeff
<< 16);
530 /* Setup horizontal resizing */
531 calc_resize_coeffs(params
->video
.in_width
,
532 params
->video
.out_width
,
533 &resize_coeff
, &downsize_coeff
);
534 reg
|= (downsize_coeff
<< 14) | resize_coeff
;
536 /* Setup color space conversion */
537 in_fmt
= format_to_colorspace(params
->video
.in_pixel_fmt
);
538 out_fmt
= format_to_colorspace(params
->video
.out_pixel_fmt
);
541 * Colourspace conversion unsupported yet - see _init_csc() in
544 if (in_fmt
!= out_fmt
) {
545 dev_err(ipu
->dev
, "Colourspace conversion unsupported!\n");
549 idmac_write_icreg(ipu
, reg
, IC_PRP_ENC_RSC
);
551 ic_conf
= idmac_read_icreg(ipu
, IC_CONF
);
554 ic_conf
&= ~IC_CONF_RWS_EN
;
556 ic_conf
|= IC_CONF_RWS_EN
;
558 idmac_write_icreg(ipu
, ic_conf
, IC_CONF
);
563 static uint32_t dma_param_addr(uint32_t dma_ch
)
565 /* Channel Parameter Memory */
566 return 0x10000 | (dma_ch
<< 4);
569 static void ipu_channel_set_priority(struct ipu
*ipu
, enum ipu_channel channel
,
572 u32 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_PRI
);
575 reg
|= 1UL << channel
;
577 reg
&= ~(1UL << channel
);
579 idmac_write_icreg(ipu
, reg
, IDMAC_CHA_PRI
);
584 static uint32_t ipu_channel_conf_mask(enum ipu_channel channel
)
591 mask
= IPU_CONF_CSI_EN
| IPU_CONF_IC_EN
;
595 mask
= IPU_CONF_SDC_EN
| IPU_CONF_DI_EN
;
606 * ipu_enable_channel() - enable an IPU channel.
607 * @idmac: IPU DMAC context.
608 * @ichan: IDMAC channel.
609 * @return: 0 on success or negative error code on failure.
611 static int ipu_enable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
613 struct ipu
*ipu
= to_ipu(idmac
);
614 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
618 spin_lock_irqsave(&ipu
->lock
, flags
);
620 /* Reset to buffer 0 */
621 idmac_write_ipureg(ipu
, 1UL << channel
, IPU_CHA_CUR_BUF
);
622 ichan
->active_buffer
= 0;
623 ichan
->status
= IPU_CHANNEL_ENABLED
;
629 ipu_channel_set_priority(ipu
, channel
, true);
634 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
636 idmac_write_icreg(ipu
, reg
| (1UL << channel
), IDMAC_CHA_EN
);
638 ipu_ic_enable_task(ipu
, channel
);
640 spin_unlock_irqrestore(&ipu
->lock
, flags
);
645 * ipu_init_channel_buffer() - initialize a buffer for logical IPU channel.
646 * @ichan: IDMAC channel.
647 * @pixel_fmt: pixel format of buffer. Pixel format is a FOURCC ASCII code.
648 * @width: width of buffer in pixels.
649 * @height: height of buffer in pixels.
650 * @stride: stride length of buffer in pixels.
651 * @rot_mode: rotation mode of buffer. A rotation setting other than
652 * IPU_ROTATE_VERT_FLIP should only be used for input buffers of
654 * @phyaddr_0: buffer 0 physical address.
655 * @phyaddr_1: buffer 1 physical address. Setting this to a value other than
656 * NULL enables double buffering mode.
657 * @return: 0 on success or negative error code on failure.
659 static int ipu_init_channel_buffer(struct idmac_channel
*ichan
,
660 enum pixel_fmt pixel_fmt
,
661 uint16_t width
, uint16_t height
,
663 enum ipu_rotate_mode rot_mode
,
664 dma_addr_t phyaddr_0
, dma_addr_t phyaddr_1
)
666 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
667 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
668 struct ipu
*ipu
= to_ipu(idmac
);
669 union chan_param_mem params
= {};
672 uint32_t stride_bytes
;
674 stride_bytes
= stride
* bytes_per_pixel(pixel_fmt
);
676 if (stride_bytes
% 4) {
678 "Stride length must be 32-bit aligned, stride = %d, bytes = %d\n",
679 stride
, stride_bytes
);
683 /* IC channel's stride must be a multiple of 8 pixels */
684 if ((channel
<= IDMAC_IC_13
) && (stride
% 8)) {
685 dev_err(ipu
->dev
, "Stride must be 8 pixel multiple\n");
689 /* Build parameter memory data for DMA channel */
690 ipu_ch_param_set_size(¶ms
, pixel_fmt
, width
, height
, stride_bytes
);
691 ipu_ch_param_set_buffer(¶ms
, phyaddr_0
, phyaddr_1
);
692 ipu_ch_param_set_rotation(¶ms
, rot_mode
);
693 /* Some channels (rotation) have restriction on burst length */
695 case IDMAC_IC_7
: /* Hangs with burst 8, 16, other values
696 invalid - Table 44-30 */
698 ipu_ch_param_set_burst_size(¶ms, 8);
703 /* In original code only IPU_PIX_FMT_RGB565 was setting burst */
704 ipu_ch_param_set_burst_size(¶ms
, 16);
711 spin_lock_irqsave(&ipu
->lock
, flags
);
713 ipu_write_param_mem(dma_param_addr(channel
), (uint32_t *)¶ms
, 10);
715 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
718 reg
|= 1UL << channel
;
720 reg
&= ~(1UL << channel
);
722 idmac_write_ipureg(ipu
, reg
, IPU_CHA_DB_MODE_SEL
);
724 ichan
->status
= IPU_CHANNEL_READY
;
726 spin_unlock_irqrestore(&ipu
->lock
, flags
);
732 * ipu_select_buffer() - mark a channel's buffer as ready.
733 * @channel: channel ID.
734 * @buffer_n: buffer number to mark ready.
736 static void ipu_select_buffer(enum ipu_channel channel
, int buffer_n
)
738 /* No locking - this is a write-one-to-set register, cleared by IPU */
740 /* Mark buffer 0 as ready. */
741 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF0_RDY
);
743 /* Mark buffer 1 as ready. */
744 idmac_write_ipureg(&ipu_data
, 1UL << channel
, IPU_CHA_BUF1_RDY
);
748 * ipu_update_channel_buffer() - update physical address of a channel buffer.
749 * @ichan: IDMAC channel.
750 * @buffer_n: buffer number to update.
751 * 0 or 1 are the only valid values.
752 * @phyaddr: buffer physical address.
754 /* Called under spin_lock(_irqsave)(&ichan->lock) */
755 static void ipu_update_channel_buffer(struct idmac_channel
*ichan
,
756 int buffer_n
, dma_addr_t phyaddr
)
758 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
762 spin_lock_irqsave(&ipu_data
.lock
, flags
);
765 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
766 if (reg
& (1UL << channel
)) {
767 ipu_ic_disable_task(&ipu_data
, channel
);
768 ichan
->status
= IPU_CHANNEL_READY
;
771 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 0) */
772 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
773 0x0008UL
, IPU_IMA_ADDR
);
774 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
776 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
777 if (reg
& (1UL << channel
)) {
778 ipu_ic_disable_task(&ipu_data
, channel
);
779 ichan
->status
= IPU_CHANNEL_READY
;
782 /* Check if double-buffering is already enabled */
783 reg
= idmac_read_ipureg(&ipu_data
, IPU_CHA_DB_MODE_SEL
);
785 if (!(reg
& (1UL << channel
)))
786 idmac_write_ipureg(&ipu_data
, reg
| (1UL << channel
),
787 IPU_CHA_DB_MODE_SEL
);
789 /* 44.3.3.1.9 - Row Number 1 (WORD1, offset 1) */
790 idmac_write_ipureg(&ipu_data
, dma_param_addr(channel
) +
791 0x0009UL
, IPU_IMA_ADDR
);
792 idmac_write_ipureg(&ipu_data
, phyaddr
, IPU_IMA_DATA
);
795 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
798 /* Called under spin_lock_irqsave(&ichan->lock) */
799 static int ipu_submit_buffer(struct idmac_channel
*ichan
,
800 struct idmac_tx_desc
*desc
, struct scatterlist
*sg
, int buf_idx
)
802 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
803 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
805 if (async_tx_test_ack(&desc
->txd
))
809 * On first invocation this shouldn't be necessary, the call to
810 * ipu_init_channel_buffer() above will set addresses for us, so we
811 * could make it conditional on status >= IPU_CHANNEL_ENABLED, but
812 * doing it again shouldn't hurt either.
814 ipu_update_channel_buffer(ichan
, buf_idx
, sg_dma_address(sg
));
816 ipu_select_buffer(chan_id
, buf_idx
);
817 dev_dbg(dev
, "Updated sg %p on channel 0x%x buffer %d\n",
818 sg
, chan_id
, buf_idx
);
823 /* Called under spin_lock_irqsave(&ichan->lock) */
824 static int ipu_submit_channel_buffers(struct idmac_channel
*ichan
,
825 struct idmac_tx_desc
*desc
)
827 struct scatterlist
*sg
;
830 for (i
= 0, sg
= desc
->sg
; i
< 2 && sg
; i
++) {
834 ret
= ipu_submit_buffer(ichan
, desc
, sg
, i
);
845 static dma_cookie_t
idmac_tx_submit(struct dma_async_tx_descriptor
*tx
)
847 struct idmac_tx_desc
*desc
= to_tx_desc(tx
);
848 struct idmac_channel
*ichan
= to_idmac_chan(tx
->chan
);
849 struct idmac
*idmac
= to_idmac(tx
->chan
->device
);
850 struct ipu
*ipu
= to_ipu(idmac
);
851 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
857 if (!list_empty(&desc
->list
)) {
858 /* The descriptor doesn't belong to client */
859 dev_err(dev
, "Descriptor %p not prepared!\n", tx
);
863 mutex_lock(&ichan
->chan_mutex
);
865 async_tx_clear_ack(tx
);
867 if (ichan
->status
< IPU_CHANNEL_READY
) {
868 struct idmac_video_param
*video
= &ichan
->params
.video
;
870 * Initial buffer assignment - the first two sg-entries from
871 * the descriptor will end up in the IDMAC buffers
873 dma_addr_t dma_1
= sg_is_last(desc
->sg
) ? 0 :
874 sg_dma_address(&desc
->sg
[1]);
876 WARN_ON(ichan
->sg
[0] || ichan
->sg
[1]);
878 cookie
= ipu_init_channel_buffer(ichan
,
879 video
->out_pixel_fmt
,
884 sg_dma_address(&desc
->sg
[0]),
890 dev_dbg(dev
, "Submitting sg %p\n", &desc
->sg
[0]);
892 cookie
= ichan
->dma_chan
.cookie
;
897 /* from dmaengine.h: "last cookie value returned to client" */
898 ichan
->dma_chan
.cookie
= cookie
;
901 /* ipu->lock can be taken under ichan->lock, but not v.v. */
902 spin_lock_irqsave(&ichan
->lock
, flags
);
904 list_add_tail(&desc
->list
, &ichan
->queue
);
905 /* submit_buffers() atomically verifies and fills empty sg slots */
906 ret
= ipu_submit_channel_buffers(ichan
, desc
);
908 spin_unlock_irqrestore(&ichan
->lock
, flags
);
915 if (ichan
->status
< IPU_CHANNEL_ENABLED
) {
916 ret
= ipu_enable_channel(idmac
, ichan
);
927 spin_lock_irqsave(&ichan
->lock
, flags
);
928 list_del_init(&desc
->list
);
929 spin_unlock_irqrestore(&ichan
->lock
, flags
);
931 ichan
->dma_chan
.cookie
= cookie
;
935 mutex_unlock(&ichan
->chan_mutex
);
940 /* Called with ichan->chan_mutex held */
941 static int idmac_desc_alloc(struct idmac_channel
*ichan
, int n
)
943 struct idmac_tx_desc
*desc
= vmalloc(n
* sizeof(struct idmac_tx_desc
));
944 struct idmac
*idmac
= to_idmac(ichan
->dma_chan
.device
);
949 /* No interrupts, just disable the tasklet for a moment */
950 tasklet_disable(&to_ipu(idmac
)->tasklet
);
952 ichan
->n_tx_desc
= n
;
954 INIT_LIST_HEAD(&ichan
->queue
);
955 INIT_LIST_HEAD(&ichan
->free_list
);
958 struct dma_async_tx_descriptor
*txd
= &desc
->txd
;
960 memset(txd
, 0, sizeof(*txd
));
961 dma_async_tx_descriptor_init(txd
, &ichan
->dma_chan
);
962 txd
->tx_submit
= idmac_tx_submit
;
964 list_add(&desc
->list
, &ichan
->free_list
);
969 tasklet_enable(&to_ipu(idmac
)->tasklet
);
975 * ipu_init_channel() - initialize an IPU channel.
976 * @idmac: IPU DMAC context.
977 * @ichan: pointer to the channel object.
978 * @return 0 on success or negative error code on failure.
980 static int ipu_init_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
982 union ipu_channel_param
*params
= &ichan
->params
;
984 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
987 struct ipu
*ipu
= to_ipu(idmac
);
988 int ret
= 0, n_desc
= 0;
990 dev_dbg(ipu
->dev
, "init channel = %d\n", channel
);
992 if (channel
!= IDMAC_SDC_0
&& channel
!= IDMAC_SDC_1
&&
993 channel
!= IDMAC_IC_7
)
996 spin_lock_irqsave(&ipu
->lock
, flags
);
1001 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1002 idmac_write_icreg(ipu
, reg
& ~IC_CONF_CSI_MEM_WR_EN
, IC_CONF
);
1006 reg
= idmac_read_ipureg(ipu
, IPU_FS_PROC_FLOW
);
1007 idmac_write_ipureg(ipu
, reg
& ~FS_ENC_IN_VALID
, IPU_FS_PROC_FLOW
);
1008 ret
= ipu_ic_init_prpenc(ipu
, params
, true);
1017 ipu
->channel_init_mask
|= 1L << channel
;
1019 /* Enable IPU sub module */
1020 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) |
1021 ipu_channel_conf_mask(channel
);
1022 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1024 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1026 if (n_desc
&& !ichan
->desc
)
1027 ret
= idmac_desc_alloc(ichan
, n_desc
);
1029 dump_idmac_reg(ipu
);
1035 * ipu_uninit_channel() - uninitialize an IPU channel.
1036 * @idmac: IPU DMAC context.
1037 * @ichan: pointer to the channel object.
1039 static void ipu_uninit_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
)
1041 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1042 unsigned long flags
;
1044 unsigned long chan_mask
= 1UL << channel
;
1046 struct ipu
*ipu
= to_ipu(idmac
);
1048 spin_lock_irqsave(&ipu
->lock
, flags
);
1050 if (!(ipu
->channel_init_mask
& chan_mask
)) {
1051 dev_err(ipu
->dev
, "Channel already uninitialized %d\n",
1053 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1057 /* Reset the double buffer */
1058 reg
= idmac_read_ipureg(ipu
, IPU_CHA_DB_MODE_SEL
);
1059 idmac_write_ipureg(ipu
, reg
& ~chan_mask
, IPU_CHA_DB_MODE_SEL
);
1061 ichan
->sec_chan_en
= false;
1065 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1066 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_RWS_EN
| IC_CONF_PRPENC_EN
),
1070 reg
= idmac_read_icreg(ipu
, IC_CONF
);
1071 idmac_write_icreg(ipu
, reg
& ~(IC_CONF_PRPENC_EN
| IC_CONF_PRPENC_CSC1
),
1080 ipu
->channel_init_mask
&= ~(1L << channel
);
1082 ipu_conf
= idmac_read_ipureg(ipu
, IPU_CONF
) &
1083 ~ipu_channel_conf_mask(channel
);
1084 idmac_write_ipureg(ipu
, ipu_conf
, IPU_CONF
);
1086 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1088 ichan
->n_tx_desc
= 0;
1094 * ipu_disable_channel() - disable an IPU channel.
1095 * @idmac: IPU DMAC context.
1096 * @ichan: channel object pointer.
1097 * @wait_for_stop: flag to set whether to wait for channel end of frame or
1098 * return immediately.
1099 * @return: 0 on success or negative error code on failure.
1101 static int ipu_disable_channel(struct idmac
*idmac
, struct idmac_channel
*ichan
,
1104 enum ipu_channel channel
= ichan
->dma_chan
.chan_id
;
1105 struct ipu
*ipu
= to_ipu(idmac
);
1107 unsigned long flags
;
1108 unsigned long chan_mask
= 1UL << channel
;
1109 unsigned int timeout
;
1111 if (wait_for_stop
&& channel
!= IDMAC_SDC_1
&& channel
!= IDMAC_SDC_0
) {
1113 /* This waiting always fails. Related to spurious irq problem */
1114 while ((idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
) & chan_mask
) ||
1115 (ipu_channel_status(ipu
, channel
) == TASK_STAT_ACTIVE
)) {
1121 "Warning: timeout waiting for channel %u to "
1122 "stop: buf0_rdy = 0x%08X, buf1_rdy = 0x%08X, "
1123 "busy = 0x%08X, tstat = 0x%08X\n", channel
,
1124 idmac_read_ipureg(ipu
, IPU_CHA_BUF0_RDY
),
1125 idmac_read_ipureg(ipu
, IPU_CHA_BUF1_RDY
),
1126 idmac_read_icreg(ipu
, IDMAC_CHA_BUSY
),
1127 idmac_read_ipureg(ipu
, IPU_TASKS_STAT
));
1131 dev_dbg(ipu
->dev
, "timeout = %d * 10ms\n", 40 - timeout
);
1133 /* SDC BG and FG must be disabled before DMA is disabled */
1134 if (wait_for_stop
&& (channel
== IDMAC_SDC_0
||
1135 channel
== IDMAC_SDC_1
)) {
1137 timeout
&& !ipu_irq_status(ichan
->eof_irq
); timeout
--)
1141 spin_lock_irqsave(&ipu
->lock
, flags
);
1143 /* Disable IC task */
1144 ipu_ic_disable_task(ipu
, channel
);
1146 /* Disable DMA channel(s) */
1147 reg
= idmac_read_icreg(ipu
, IDMAC_CHA_EN
);
1148 idmac_write_icreg(ipu
, reg
& ~chan_mask
, IDMAC_CHA_EN
);
1150 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1155 static struct scatterlist
*idmac_sg_next(struct idmac_channel
*ichan
,
1156 struct idmac_tx_desc
**desc
, struct scatterlist
*sg
)
1158 struct scatterlist
*sgnew
= sg
? sg_next(sg
) : NULL
;
1161 /* next sg-element in this list */
1164 if ((*desc
)->list
.next
== &ichan
->queue
)
1165 /* No more descriptors on the queue */
1168 /* Fetch next descriptor */
1169 *desc
= list_entry((*desc
)->list
.next
, struct idmac_tx_desc
, list
);
1174 * We have several possibilities here:
1175 * current BUF next BUF
1177 * not last sg next not last sg
1178 * not last sg next last sg
1179 * last sg first sg from next descriptor
1182 * Besides, the descriptor queue might be empty or not. We process all these
1185 static irqreturn_t
idmac_interrupt(int irq
, void *dev_id
)
1187 struct idmac_channel
*ichan
= dev_id
;
1188 struct device
*dev
= &ichan
->dma_chan
.dev
->device
;
1189 unsigned int chan_id
= ichan
->dma_chan
.chan_id
;
1190 struct scatterlist
**sg
, *sgnext
, *sgnew
= NULL
;
1191 /* Next transfer descriptor */
1192 struct idmac_tx_desc
*desc
, *descnew
;
1193 dma_async_tx_callback callback
;
1194 void *callback_param
;
1196 u32 ready0
, ready1
, curbuf
, err
;
1197 unsigned long flags
;
1199 /* IDMAC has cleared the respective BUFx_RDY bit, we manage the buffer */
1201 dev_dbg(dev
, "IDMAC irq %d, buf %d\n", irq
, ichan
->active_buffer
);
1203 spin_lock_irqsave(&ipu_data
.lock
, flags
);
1205 ready0
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF0_RDY
);
1206 ready1
= idmac_read_ipureg(&ipu_data
, IPU_CHA_BUF1_RDY
);
1207 curbuf
= idmac_read_ipureg(&ipu_data
, IPU_CHA_CUR_BUF
);
1208 err
= idmac_read_ipureg(&ipu_data
, IPU_INT_STAT_4
);
1210 if (err
& (1 << chan_id
)) {
1211 idmac_write_ipureg(&ipu_data
, 1 << chan_id
, IPU_INT_STAT_4
);
1212 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1215 * ichan->sg[0] = ichan->sg[1] = NULL;
1216 * you can force channel re-enable on the next tx_submit(), but
1217 * this is dirty - think about descriptors with multiple
1220 dev_warn(dev
, "NFB4EOF on channel %d, ready %x, %x, cur %x\n",
1221 chan_id
, ready0
, ready1
, curbuf
);
1224 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1226 /* Other interrupts do not interfere with this channel */
1227 spin_lock(&ichan
->lock
);
1228 if (unlikely((ichan
->active_buffer
&& (ready1
>> chan_id
) & 1) ||
1229 (!ichan
->active_buffer
&& (ready0
>> chan_id
) & 1)
1231 spin_unlock(&ichan
->lock
);
1233 "IRQ with active buffer still ready on channel %x, "
1234 "active %d, ready %x, %x!\n", chan_id
,
1235 ichan
->active_buffer
, ready0
, ready1
);
1239 if (unlikely(list_empty(&ichan
->queue
))) {
1240 ichan
->sg
[ichan
->active_buffer
] = NULL
;
1241 spin_unlock(&ichan
->lock
);
1243 "IRQ without queued buffers on channel %x, active %d, "
1244 "ready %x, %x!\n", chan_id
,
1245 ichan
->active_buffer
, ready0
, ready1
);
1250 * active_buffer is a software flag, it shows which buffer we are
1251 * currently expecting back from the hardware, IDMAC should be
1252 * processing the other buffer already
1254 sg
= &ichan
->sg
[ichan
->active_buffer
];
1255 sgnext
= ichan
->sg
[!ichan
->active_buffer
];
1258 spin_unlock(&ichan
->lock
);
1262 desc
= list_entry(ichan
->queue
.next
, struct idmac_tx_desc
, list
);
1265 dev_dbg(dev
, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n",
1266 irq
, sg_dma_address(*sg
), sgnext
? sg_dma_address(sgnext
) : 0, ichan
->active_buffer
, curbuf
);
1268 /* Find the descriptor of sgnext */
1269 sgnew
= idmac_sg_next(ichan
, &descnew
, *sg
);
1270 if (sgnext
!= sgnew
)
1271 dev_err(dev
, "Submitted buffer %p, next buffer %p\n", sgnext
, sgnew
);
1274 * if sgnext == NULL sg must be the last element in a scatterlist and
1275 * queue must be empty
1277 if (unlikely(!sgnext
)) {
1278 if (!WARN_ON(sg_next(*sg
)))
1279 dev_dbg(dev
, "Underrun on channel %x\n", chan_id
);
1280 ichan
->sg
[!ichan
->active_buffer
] = sgnew
;
1282 if (unlikely(sgnew
)) {
1283 ipu_submit_buffer(ichan
, descnew
, sgnew
, !ichan
->active_buffer
);
1285 spin_lock_irqsave(&ipu_data
.lock
, flags
);
1286 ipu_ic_disable_task(&ipu_data
, chan_id
);
1287 spin_unlock_irqrestore(&ipu_data
.lock
, flags
);
1288 ichan
->status
= IPU_CHANNEL_READY
;
1289 /* Continue to check for complete descriptor */
1293 /* Calculate and submit the next sg element */
1294 sgnew
= idmac_sg_next(ichan
, &descnew
, sgnew
);
1296 if (unlikely(!sg_next(*sg
)) || !sgnext
) {
1298 * Last element in scatterlist done, remove from the queue,
1299 * _init for debugging
1301 list_del_init(&desc
->list
);
1307 if (likely(sgnew
) &&
1308 ipu_submit_buffer(ichan
, descnew
, sgnew
, ichan
->active_buffer
) < 0) {
1309 callback
= descnew
->txd
.callback
;
1310 callback_param
= descnew
->txd
.callback_param
;
1311 spin_unlock(&ichan
->lock
);
1313 callback(callback_param
);
1314 spin_lock(&ichan
->lock
);
1317 /* Flip the active buffer - even if update above failed */
1318 ichan
->active_buffer
= !ichan
->active_buffer
;
1320 ichan
->completed
= desc
->txd
.cookie
;
1322 callback
= desc
->txd
.callback
;
1323 callback_param
= desc
->txd
.callback_param
;
1325 spin_unlock(&ichan
->lock
);
1327 if (done
&& (desc
->txd
.flags
& DMA_PREP_INTERRUPT
) && callback
)
1328 callback(callback_param
);
1333 static void ipu_gc_tasklet(unsigned long arg
)
1335 struct ipu
*ipu
= (struct ipu
*)arg
;
1338 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1339 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1340 struct idmac_tx_desc
*desc
;
1341 unsigned long flags
;
1342 struct scatterlist
*sg
;
1345 for (j
= 0; j
< ichan
->n_tx_desc
; j
++) {
1346 desc
= ichan
->desc
+ j
;
1347 spin_lock_irqsave(&ichan
->lock
, flags
);
1348 if (async_tx_test_ack(&desc
->txd
)) {
1349 list_move(&desc
->list
, &ichan
->free_list
);
1350 for_each_sg(desc
->sg
, sg
, desc
->sg_len
, k
) {
1351 if (ichan
->sg
[0] == sg
)
1352 ichan
->sg
[0] = NULL
;
1353 else if (ichan
->sg
[1] == sg
)
1354 ichan
->sg
[1] = NULL
;
1356 async_tx_clear_ack(&desc
->txd
);
1358 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1363 /* Allocate and initialise a transfer descriptor. */
1364 static struct dma_async_tx_descriptor
*idmac_prep_slave_sg(struct dma_chan
*chan
,
1365 struct scatterlist
*sgl
, unsigned int sg_len
,
1366 enum dma_data_direction direction
, unsigned long tx_flags
)
1368 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1369 struct idmac_tx_desc
*desc
= NULL
;
1370 struct dma_async_tx_descriptor
*txd
= NULL
;
1371 unsigned long flags
;
1373 /* We only can handle these three channels so far */
1374 if (chan
->chan_id
!= IDMAC_SDC_0
&& chan
->chan_id
!= IDMAC_SDC_1
&&
1375 chan
->chan_id
!= IDMAC_IC_7
)
1378 if (direction
!= DMA_FROM_DEVICE
&& direction
!= DMA_TO_DEVICE
) {
1379 dev_err(chan
->device
->dev
, "Invalid DMA direction %d!\n", direction
);
1383 mutex_lock(&ichan
->chan_mutex
);
1385 spin_lock_irqsave(&ichan
->lock
, flags
);
1386 if (!list_empty(&ichan
->free_list
)) {
1387 desc
= list_entry(ichan
->free_list
.next
,
1388 struct idmac_tx_desc
, list
);
1390 list_del_init(&desc
->list
);
1392 desc
->sg_len
= sg_len
;
1395 txd
->flags
= tx_flags
;
1397 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1399 mutex_unlock(&ichan
->chan_mutex
);
1401 tasklet_schedule(&to_ipu(to_idmac(chan
->device
))->tasklet
);
1406 /* Re-select the current buffer and re-activate the channel */
1407 static void idmac_issue_pending(struct dma_chan
*chan
)
1409 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1410 struct idmac
*idmac
= to_idmac(chan
->device
);
1411 struct ipu
*ipu
= to_ipu(idmac
);
1412 unsigned long flags
;
1414 /* This is not always needed, but doesn't hurt either */
1415 spin_lock_irqsave(&ipu
->lock
, flags
);
1416 ipu_select_buffer(chan
->chan_id
, ichan
->active_buffer
);
1417 spin_unlock_irqrestore(&ipu
->lock
, flags
);
1420 * Might need to perform some parts of initialisation from
1421 * ipu_enable_channel(), but not all, we do not want to reset to buffer
1422 * 0, don't need to set priority again either, but re-enabling the task
1423 * and the channel might be a good idea.
1427 static int __idmac_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
1430 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1431 struct idmac
*idmac
= to_idmac(chan
->device
);
1432 unsigned long flags
;
1435 /* Only supports DMA_TERMINATE_ALL */
1436 if (cmd
!= DMA_TERMINATE_ALL
)
1439 ipu_disable_channel(idmac
, ichan
,
1440 ichan
->status
>= IPU_CHANNEL_ENABLED
);
1442 tasklet_disable(&to_ipu(idmac
)->tasklet
);
1444 /* ichan->queue is modified in ISR, have to spinlock */
1445 spin_lock_irqsave(&ichan
->lock
, flags
);
1446 list_splice_init(&ichan
->queue
, &ichan
->free_list
);
1449 for (i
= 0; i
< ichan
->n_tx_desc
; i
++) {
1450 struct idmac_tx_desc
*desc
= ichan
->desc
+ i
;
1451 if (list_empty(&desc
->list
))
1452 /* Descriptor was prepared, but not submitted */
1453 list_add(&desc
->list
, &ichan
->free_list
);
1455 async_tx_clear_ack(&desc
->txd
);
1458 ichan
->sg
[0] = NULL
;
1459 ichan
->sg
[1] = NULL
;
1460 spin_unlock_irqrestore(&ichan
->lock
, flags
);
1462 tasklet_enable(&to_ipu(idmac
)->tasklet
);
1464 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1469 static int idmac_control(struct dma_chan
*chan
, enum dma_ctrl_cmd cmd
,
1472 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1475 mutex_lock(&ichan
->chan_mutex
);
1477 ret
= __idmac_control(chan
, cmd
, arg
);
1479 mutex_unlock(&ichan
->chan_mutex
);
1485 static irqreturn_t
ic_sof_irq(int irq
, void *dev_id
)
1487 struct idmac_channel
*ichan
= dev_id
;
1488 printk(KERN_DEBUG
"Got SOF IRQ %d on Channel %d\n",
1489 irq
, ichan
->dma_chan
.chan_id
);
1490 disable_irq_nosync(irq
);
1494 static irqreturn_t
ic_eof_irq(int irq
, void *dev_id
)
1496 struct idmac_channel
*ichan
= dev_id
;
1497 printk(KERN_DEBUG
"Got EOF IRQ %d on Channel %d\n",
1498 irq
, ichan
->dma_chan
.chan_id
);
1499 disable_irq_nosync(irq
);
1503 static int ic_sof
= -EINVAL
, ic_eof
= -EINVAL
;
1506 static int idmac_alloc_chan_resources(struct dma_chan
*chan
)
1508 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1509 struct idmac
*idmac
= to_idmac(chan
->device
);
1512 /* dmaengine.c now guarantees to only offer free channels */
1513 BUG_ON(chan
->client_count
> 1);
1514 WARN_ON(ichan
->status
!= IPU_CHANNEL_FREE
);
1517 ichan
->completed
= -ENXIO
;
1519 ret
= ipu_irq_map(chan
->chan_id
);
1523 ichan
->eof_irq
= ret
;
1526 * Important to first disable the channel, because maybe someone
1527 * used it before us, e.g., the bootloader
1529 ipu_disable_channel(idmac
, ichan
, true);
1531 ret
= ipu_init_channel(idmac
, ichan
);
1535 ret
= request_irq(ichan
->eof_irq
, idmac_interrupt
, 0,
1536 ichan
->eof_name
, ichan
);
1541 if (chan
->chan_id
== IDMAC_IC_7
) {
1542 ic_sof
= ipu_irq_map(69);
1544 request_irq(ic_sof
, ic_sof_irq
, 0, "IC SOF", ichan
);
1545 ic_eof
= ipu_irq_map(70);
1547 request_irq(ic_eof
, ic_eof_irq
, 0, "IC EOF", ichan
);
1551 ichan
->status
= IPU_CHANNEL_INITIALIZED
;
1553 dev_dbg(&chan
->dev
->device
, "Found channel 0x%x, irq %d\n",
1554 chan
->chan_id
, ichan
->eof_irq
);
1559 ipu_uninit_channel(idmac
, ichan
);
1561 ipu_irq_unmap(chan
->chan_id
);
1566 static void idmac_free_chan_resources(struct dma_chan
*chan
)
1568 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1569 struct idmac
*idmac
= to_idmac(chan
->device
);
1571 mutex_lock(&ichan
->chan_mutex
);
1573 __idmac_control(chan
, DMA_TERMINATE_ALL
, 0);
1575 if (ichan
->status
> IPU_CHANNEL_FREE
) {
1577 if (chan
->chan_id
== IDMAC_IC_7
) {
1579 free_irq(ic_sof
, ichan
);
1584 free_irq(ic_eof
, ichan
);
1590 free_irq(ichan
->eof_irq
, ichan
);
1591 ipu_irq_unmap(chan
->chan_id
);
1594 ichan
->status
= IPU_CHANNEL_FREE
;
1596 ipu_uninit_channel(idmac
, ichan
);
1598 mutex_unlock(&ichan
->chan_mutex
);
1600 tasklet_schedule(&to_ipu(idmac
)->tasklet
);
1603 static enum dma_status
idmac_tx_status(struct dma_chan
*chan
,
1604 dma_cookie_t cookie
, struct dma_tx_state
*txstate
)
1606 struct idmac_channel
*ichan
= to_idmac_chan(chan
);
1608 dma_set_tx_state(txstate
, ichan
->completed
, chan
->cookie
, 0);
1609 if (cookie
!= chan
->cookie
)
1614 static int __init
ipu_idmac_init(struct ipu
*ipu
)
1616 struct idmac
*idmac
= &ipu
->idmac
;
1617 struct dma_device
*dma
= &idmac
->dma
;
1620 dma_cap_set(DMA_SLAVE
, dma
->cap_mask
);
1621 dma_cap_set(DMA_PRIVATE
, dma
->cap_mask
);
1623 /* Compulsory common fields */
1624 dma
->dev
= ipu
->dev
;
1625 dma
->device_alloc_chan_resources
= idmac_alloc_chan_resources
;
1626 dma
->device_free_chan_resources
= idmac_free_chan_resources
;
1627 dma
->device_tx_status
= idmac_tx_status
;
1628 dma
->device_issue_pending
= idmac_issue_pending
;
1630 /* Compulsory for DMA_SLAVE fields */
1631 dma
->device_prep_slave_sg
= idmac_prep_slave_sg
;
1632 dma
->device_control
= idmac_control
;
1634 INIT_LIST_HEAD(&dma
->channels
);
1635 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1636 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1637 struct dma_chan
*dma_chan
= &ichan
->dma_chan
;
1639 spin_lock_init(&ichan
->lock
);
1640 mutex_init(&ichan
->chan_mutex
);
1642 ichan
->status
= IPU_CHANNEL_FREE
;
1643 ichan
->sec_chan_en
= false;
1644 ichan
->completed
= -ENXIO
;
1645 snprintf(ichan
->eof_name
, sizeof(ichan
->eof_name
), "IDMAC EOF %d", i
);
1647 dma_chan
->device
= &idmac
->dma
;
1648 dma_chan
->cookie
= 1;
1649 dma_chan
->chan_id
= i
;
1650 list_add_tail(&dma_chan
->device_node
, &dma
->channels
);
1653 idmac_write_icreg(ipu
, 0x00000070, IDMAC_CONF
);
1655 return dma_async_device_register(&idmac
->dma
);
1658 static void __exit
ipu_idmac_exit(struct ipu
*ipu
)
1661 struct idmac
*idmac
= &ipu
->idmac
;
1663 for (i
= 0; i
< IPU_CHANNELS_NUM
; i
++) {
1664 struct idmac_channel
*ichan
= ipu
->channel
+ i
;
1666 idmac_control(&ichan
->dma_chan
, DMA_TERMINATE_ALL
, 0);
1667 idmac_prep_slave_sg(&ichan
->dma_chan
, NULL
, 0, DMA_NONE
, 0);
1670 dma_async_device_unregister(&idmac
->dma
);
1673 /*****************************************************************************
1674 * IPU common probe / remove
1677 static int __init
ipu_probe(struct platform_device
*pdev
)
1679 struct ipu_platform_data
*pdata
= pdev
->dev
.platform_data
;
1680 struct resource
*mem_ipu
, *mem_ic
;
1683 spin_lock_init(&ipu_data
.lock
);
1685 mem_ipu
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1686 mem_ic
= platform_get_resource(pdev
, IORESOURCE_MEM
, 1);
1687 if (!pdata
|| !mem_ipu
|| !mem_ic
)
1690 ipu_data
.dev
= &pdev
->dev
;
1692 platform_set_drvdata(pdev
, &ipu_data
);
1694 ret
= platform_get_irq(pdev
, 0);
1698 ipu_data
.irq_fn
= ret
;
1699 ret
= platform_get_irq(pdev
, 1);
1703 ipu_data
.irq_err
= ret
;
1704 ipu_data
.irq_base
= pdata
->irq_base
;
1706 dev_dbg(&pdev
->dev
, "fn irq %u, err irq %u, irq-base %u\n",
1707 ipu_data
.irq_fn
, ipu_data
.irq_err
, ipu_data
.irq_base
);
1709 /* Remap IPU common registers */
1710 ipu_data
.reg_ipu
= ioremap(mem_ipu
->start
, resource_size(mem_ipu
));
1711 if (!ipu_data
.reg_ipu
) {
1713 goto err_ioremap_ipu
;
1716 /* Remap Image Converter and Image DMA Controller registers */
1717 ipu_data
.reg_ic
= ioremap(mem_ic
->start
, resource_size(mem_ic
));
1718 if (!ipu_data
.reg_ic
) {
1720 goto err_ioremap_ic
;
1724 ipu_data
.ipu_clk
= clk_get(&pdev
->dev
, NULL
);
1725 if (IS_ERR(ipu_data
.ipu_clk
)) {
1726 ret
= PTR_ERR(ipu_data
.ipu_clk
);
1730 /* Make sure IPU HSP clock is running */
1731 clk_enable(ipu_data
.ipu_clk
);
1733 /* Disable all interrupts */
1734 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_1
);
1735 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_2
);
1736 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_3
);
1737 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_4
);
1738 idmac_write_ipureg(&ipu_data
, 0, IPU_INT_CTRL_5
);
1740 dev_dbg(&pdev
->dev
, "%s @ 0x%08lx, fn irq %u, err irq %u\n", pdev
->name
,
1741 (unsigned long)mem_ipu
->start
, ipu_data
.irq_fn
, ipu_data
.irq_err
);
1743 ret
= ipu_irq_attach_irq(&ipu_data
, pdev
);
1745 goto err_attach_irq
;
1747 /* Initialize DMA engine */
1748 ret
= ipu_idmac_init(&ipu_data
);
1750 goto err_idmac_init
;
1752 tasklet_init(&ipu_data
.tasklet
, ipu_gc_tasklet
, (unsigned long)&ipu_data
);
1754 ipu_data
.dev
= &pdev
->dev
;
1756 dev_dbg(ipu_data
.dev
, "IPU initialized\n");
1762 ipu_irq_detach_irq(&ipu_data
, pdev
);
1763 clk_disable(ipu_data
.ipu_clk
);
1764 clk_put(ipu_data
.ipu_clk
);
1766 iounmap(ipu_data
.reg_ic
);
1768 iounmap(ipu_data
.reg_ipu
);
1771 dev_err(&pdev
->dev
, "Failed to probe IPU: %d\n", ret
);
1775 static int __exit
ipu_remove(struct platform_device
*pdev
)
1777 struct ipu
*ipu
= platform_get_drvdata(pdev
);
1779 ipu_idmac_exit(ipu
);
1780 ipu_irq_detach_irq(ipu
, pdev
);
1781 clk_disable(ipu
->ipu_clk
);
1782 clk_put(ipu
->ipu_clk
);
1783 iounmap(ipu
->reg_ic
);
1784 iounmap(ipu
->reg_ipu
);
1785 tasklet_kill(&ipu
->tasklet
);
1786 platform_set_drvdata(pdev
, NULL
);
1792 * We need two MEM resources - with IPU-common and Image Converter registers,
1793 * including PF_CONF and IDMAC_* registers, and two IRQs - function and error
1795 static struct platform_driver ipu_platform_driver
= {
1798 .owner
= THIS_MODULE
,
1800 .remove
= __exit_p(ipu_remove
),
1803 static int __init
ipu_init(void)
1805 return platform_driver_probe(&ipu_platform_driver
, ipu_probe
);
1807 subsys_initcall(ipu_init
);
1809 MODULE_DESCRIPTION("IPU core driver");
1810 MODULE_LICENSE("GPL v2");
1811 MODULE_AUTHOR("Guennadi Liakhovetski <lg@denx.de>");
1812 MODULE_ALIAS("platform:ipu-core");