?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / drivers / gpu / ipu-v3 / ipu-image-convert.c
blob841316582ea9d3b875df48632447c61a97c8e936
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) 2012-2016 Mentor Graphics Inc.
4 *
5 * Queued image conversion support, with tiling and rotation.
6 */
7
8 #include <linux/interrupt.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/math.h>
11
12 #include <video/imx-ipu-image-convert.h>
13
14 #include "ipu-prv.h"
15
16 /*
17 * The IC Resizer has a restriction that the output frame from the
18 * resizer must be 1024 or less in both width (pixels) and height
19 * (lines).
20 *
21 * The image converter attempts to split up a conversion when
22 * the desired output (converted) frame resolution exceeds the
23 * IC resizer limit of 1024 in either dimension.
24 *
25 * If either dimension of the output frame exceeds the limit, the
26 * dimension is split into 1, 2, or 4 equal stripes, for a maximum
27 * of 4*4 or 16 tiles. A conversion is then carried out for each
28 * tile (but taking care to pass the full frame stride length to
29 * the DMA channel's parameter memory!). IDMA double-buffering is used
30 * to convert each tile back-to-back when possible (see note below
31 * when double_buffering boolean is set).
32 *
33 * Note that the input frame must be split up into the same number
34 * of tiles as the output frame:
35 *
36 * +---------+-----+
37 * +-----+---+ | A | B |
38 * | A | B | | | |
39 * +-----+---+ --> +---------+-----+
40 * | C | D | | C | D |
41 * +-----+---+ | | |
42 * +---------+-----+
43 *
44 * Clockwise 90° rotations are handled by first rescaling into a
45 * reusable temporary tile buffer and then rotating with the 8x8
46 * block rotator, writing to the correct destination:
47 *
48 * +-----+-----+
49 * | | |
50 * +-----+---+ +---------+ | C | A |
51 * | A | B | | A,B, | | | | |
52 * +-----+---+ --> | C,D | | --> | | |
53 * | C | D | +---------+ +-----+-----+
54 * +-----+---+ | D | B |
55 * | | |
56 * +-----+-----+
57 *
58 * If the 8x8 block rotator is used, horizontal or vertical flipping
59 * is done during the rotation step, otherwise flipping is done
60 * during the scaling step.
61 * With rotation or flipping, tile order changes between input and
62 * output image. Tiles are numbered row major from top left to bottom
63 * right for both input and output image.
64 */
65
66 #define MAX_STRIPES_W 4
67 #define MAX_STRIPES_H 4
68 #define MAX_TILES (MAX_STRIPES_W * MAX_STRIPES_H)
69
70 #define MIN_W 16
71 #define MIN_H 8
72 #define MAX_W 4096
73 #define MAX_H 4096
74
75 enum ipu_image_convert_type {
76 IMAGE_CONVERT_IN = 0,
77 IMAGE_CONVERT_OUT,
78 };
79
80 struct ipu_image_convert_dma_buf {
81 void *virt;
82 dma_addr_t phys;
83 unsigned long len;
84 };
85
86 struct ipu_image_convert_dma_chan {
87 int in;
88 int out;
89 int rot_in;
90 int rot_out;
91 int vdi_in_p;
92 int vdi_in;
93 int vdi_in_n;
94 };
95
96 /* dimensions of one tile */
97 struct ipu_image_tile {
98 u32 width;
99 u32 height;
100 u32 left;
101 u32 top;
102 /* size and strides are in bytes */
103 u32 size;
104 u32 stride;
105 u32 rot_stride;
106 /* start Y or packed offset of this tile */
107 u32 offset;
108 /* offset from start to tile in U plane, for planar formats */
109 u32 u_off;
110 /* offset from start to tile in V plane, for planar formats */
111 u32 v_off;
112 };
113
114 struct ipu_image_convert_image {
115 struct ipu_image base;
116 enum ipu_image_convert_type type;
117
118 const struct ipu_image_pixfmt *fmt;
119 unsigned int stride;
120
121 /* # of rows (horizontal stripes) if dest height is > 1024 */
122 unsigned int num_rows;
123 /* # of columns (vertical stripes) if dest width is > 1024 */
124 unsigned int num_cols;
125
126 struct ipu_image_tile tile[MAX_TILES];
127 };
128
129 struct ipu_image_pixfmt {
130 u32 fourcc; /* V4L2 fourcc */
131 int bpp; /* total bpp */
132 int uv_width_dec; /* decimation in width for U/V planes */
133 int uv_height_dec; /* decimation in height for U/V planes */
134 bool planar; /* planar format */
135 bool uv_swapped; /* U and V planes are swapped */
136 bool uv_packed; /* partial planar (U and V in same plane) */
137 };
138
139 struct ipu_image_convert_ctx;
140 struct ipu_image_convert_chan;
141 struct ipu_image_convert_priv;
142
143 enum eof_irq_mask {
144 EOF_IRQ_IN = BIT(0),
145 EOF_IRQ_ROT_IN = BIT(1),
146 EOF_IRQ_OUT = BIT(2),
147 EOF_IRQ_ROT_OUT = BIT(3),
148 };
149
150 #define EOF_IRQ_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT)
151 #define EOF_IRQ_ROT_COMPLETE (EOF_IRQ_IN | EOF_IRQ_OUT | \
152 EOF_IRQ_ROT_IN | EOF_IRQ_ROT_OUT)
153
154 struct ipu_image_convert_ctx {
155 struct ipu_image_convert_chan *chan;
156
157 ipu_image_convert_cb_t complete;
158 void *complete_context;
159
160 /* Source/destination image data and rotation mode */
161 struct ipu_image_convert_image in;
162 struct ipu_image_convert_image out;
163 struct ipu_ic_csc csc;
164 enum ipu_rotate_mode rot_mode;
165 u32 downsize_coeff_h;
166 u32 downsize_coeff_v;
167 u32 image_resize_coeff_h;
168 u32 image_resize_coeff_v;
169 u32 resize_coeffs_h[MAX_STRIPES_W];
170 u32 resize_coeffs_v[MAX_STRIPES_H];
171
172 /* intermediate buffer for rotation */
173 struct ipu_image_convert_dma_buf rot_intermediate[2];
174
175 /* current buffer number for double buffering */
176 int cur_buf_num;
177
178 bool aborting;
179 struct completion aborted;
180
181 /* can we use double-buffering for this conversion operation? */
182 bool double_buffering;
183 /* num_rows * num_cols */
184 unsigned int num_tiles;
185 /* next tile to process */
186 unsigned int next_tile;
187 /* where to place converted tile in dest image */
188 unsigned int out_tile_map[MAX_TILES];
189
190 /* mask of completed EOF irqs at every tile conversion */
191 enum eof_irq_mask eof_mask;
192
193 struct list_head list;
194 };
195
196 struct ipu_image_convert_chan {
197 struct ipu_image_convert_priv *priv;
198
199 enum ipu_ic_task ic_task;
200 const struct ipu_image_convert_dma_chan *dma_ch;
201
202 struct ipu_ic *ic;
203 struct ipuv3_channel *in_chan;
204 struct ipuv3_channel *out_chan;
205 struct ipuv3_channel *rotation_in_chan;
206 struct ipuv3_channel *rotation_out_chan;
207
208 /* the IPU end-of-frame irqs */
209 int in_eof_irq;
210 int rot_in_eof_irq;
211 int out_eof_irq;
212 int rot_out_eof_irq;
213
214 spinlock_t irqlock;
215
216 /* list of convert contexts */
217 struct list_head ctx_list;
218 /* queue of conversion runs */
219 struct list_head pending_q;
220 /* queue of completed runs */
221 struct list_head done_q;
222
223 /* the current conversion run */
224 struct ipu_image_convert_run *current_run;
225 };
226
227 struct ipu_image_convert_priv {
228 struct ipu_image_convert_chan chan[IC_NUM_TASKS];
229 struct ipu_soc *ipu;
230 };
231
232 static const struct ipu_image_convert_dma_chan
233 image_convert_dma_chan[IC_NUM_TASKS] = {
234 [IC_TASK_VIEWFINDER] = {
235 .in = IPUV3_CHANNEL_MEM_IC_PRP_VF,
236 .out = IPUV3_CHANNEL_IC_PRP_VF_MEM,
237 .rot_in = IPUV3_CHANNEL_MEM_ROT_VF,
238 .rot_out = IPUV3_CHANNEL_ROT_VF_MEM,
239 .vdi_in_p = IPUV3_CHANNEL_MEM_VDI_PREV,
240 .vdi_in = IPUV3_CHANNEL_MEM_VDI_CUR,
241 .vdi_in_n = IPUV3_CHANNEL_MEM_VDI_NEXT,
242 },
243 [IC_TASK_POST_PROCESSOR] = {
244 .in = IPUV3_CHANNEL_MEM_IC_PP,
245 .out = IPUV3_CHANNEL_IC_PP_MEM,
246 .rot_in = IPUV3_CHANNEL_MEM_ROT_PP,
247 .rot_out = IPUV3_CHANNEL_ROT_PP_MEM,
248 },
249 };
250
251 static const struct ipu_image_pixfmt image_convert_formats[] = {
252 {
253 .fourcc = V4L2_PIX_FMT_RGB565,
254 .bpp = 16,
255 }, {
256 .fourcc = V4L2_PIX_FMT_RGB24,
257 .bpp = 24,
258 }, {
259 .fourcc = V4L2_PIX_FMT_BGR24,
260 .bpp = 24,
261 }, {
262 .fourcc = V4L2_PIX_FMT_RGB32,
263 .bpp = 32,
264 }, {
265 .fourcc = V4L2_PIX_FMT_BGR32,
266 .bpp = 32,
267 }, {
268 .fourcc = V4L2_PIX_FMT_XRGB32,
269 .bpp = 32,
270 }, {
271 .fourcc = V4L2_PIX_FMT_XBGR32,
272 .bpp = 32,
273 }, {
274 .fourcc = V4L2_PIX_FMT_BGRX32,
275 .bpp = 32,
276 }, {
277 .fourcc = V4L2_PIX_FMT_RGBX32,
278 .bpp = 32,
279 }, {
280 .fourcc = V4L2_PIX_FMT_YUYV,
281 .bpp = 16,
282 .uv_width_dec = 2,
283 .uv_height_dec = 1,
284 }, {
285 .fourcc = V4L2_PIX_FMT_UYVY,
286 .bpp = 16,
287 .uv_width_dec = 2,
288 .uv_height_dec = 1,
289 }, {
290 .fourcc = V4L2_PIX_FMT_YUV420,
291 .bpp = 12,
292 .planar = true,
293 .uv_width_dec = 2,
294 .uv_height_dec = 2,
295 }, {
296 .fourcc = V4L2_PIX_FMT_YVU420,
297 .bpp = 12,
298 .planar = true,
299 .uv_width_dec = 2,
300 .uv_height_dec = 2,
301 .uv_swapped = true,
302 }, {
303 .fourcc = V4L2_PIX_FMT_NV12,
304 .bpp = 12,
305 .planar = true,
306 .uv_width_dec = 2,
307 .uv_height_dec = 2,
308 .uv_packed = true,
309 }, {
310 .fourcc = V4L2_PIX_FMT_YUV422P,
311 .bpp = 16,
312 .planar = true,
313 .uv_width_dec = 2,
314 .uv_height_dec = 1,
315 }, {
316 .fourcc = V4L2_PIX_FMT_NV16,
317 .bpp = 16,
318 .planar = true,
319 .uv_width_dec = 2,
320 .uv_height_dec = 1,
321 .uv_packed = true,
322 },
323 };
324
325 static const struct ipu_image_pixfmt *get_format(u32 fourcc)
326 {
327 const struct ipu_image_pixfmt *ret = NULL;
328 unsigned int i;
329
330 for (i = 0; i < ARRAY_SIZE(image_convert_formats); i++) {
331 if (image_convert_formats[i].fourcc == fourcc) {
332 ret = &image_convert_formats[i];
333 break;
334 }
335 }
336
337 return ret;
338 }
339
340 static void dump_format(struct ipu_image_convert_ctx *ctx,
341 struct ipu_image_convert_image *ic_image)
342 {
343 struct ipu_image_convert_chan *chan = ctx->chan;
344 struct ipu_image_convert_priv *priv = chan->priv;
345
346 dev_dbg(priv->ipu->dev,
347 "task %u: ctx %p: %s format: %dx%d (%dx%d tiles), %c%c%c%c\n",
348 chan->ic_task, ctx,
349 ic_image->type == IMAGE_CONVERT_OUT ? "Output" : "Input",
350 ic_image->base.pix.width, ic_image->base.pix.height,
351 ic_image->num_cols, ic_image->num_rows,
352 ic_image->fmt->fourcc & 0xff,
353 (ic_image->fmt->fourcc >> 8) & 0xff,
354 (ic_image->fmt->fourcc >> 16) & 0xff,
355 (ic_image->fmt->fourcc >> 24) & 0xff);
356 }
357
358 int ipu_image_convert_enum_format(int index, u32 *fourcc)
359 {
360 const struct ipu_image_pixfmt *fmt;
361
362 if (index >= (int)ARRAY_SIZE(image_convert_formats))
363 return -EINVAL;
364
365 /* Format found */
366 fmt = &image_convert_formats[index];
367 *fourcc = fmt->fourcc;
368 return 0;
369 }
370 EXPORT_SYMBOL_GPL(ipu_image_convert_enum_format);
371
372 static void free_dma_buf(struct ipu_image_convert_priv *priv,
373 struct ipu_image_convert_dma_buf *buf)
374 {
375 if (buf->virt)
376 dma_free_coherent(priv->ipu->dev,
377 buf->len, buf->virt, buf->phys);
378 buf->virt = NULL;
379 buf->phys = 0;
380 }
381
382 static int alloc_dma_buf(struct ipu_image_convert_priv *priv,
383 struct ipu_image_convert_dma_buf *buf,
384 int size)
385 {
386 buf->len = PAGE_ALIGN(size);
387 buf->virt = dma_alloc_coherent(priv->ipu->dev, buf->len, &buf->phys,
388 GFP_DMA | GFP_KERNEL);
389 if (!buf->virt) {
390 dev_err(priv->ipu->dev, "failed to alloc dma buffer\n");
391 return -ENOMEM;
392 }
393
394 return 0;
395 }
396
397 static inline int num_stripes(int dim)
398 {
399 return (dim - 1) / 1024 + 1;
400 }
401
402 /*
403 * Calculate downsizing coefficients, which are the same for all tiles,
404 * and initial bilinear resizing coefficients, which are used to find the
405 * best seam positions.
406 * Also determine the number of tiles necessary to guarantee that no tile
407 * is larger than 1024 pixels in either dimension at the output and between
408 * IC downsizing and main processing sections.
409 */
410 static int calc_image_resize_coefficients(struct ipu_image_convert_ctx *ctx,
411 struct ipu_image *in,
412 struct ipu_image *out)
413 {
414 u32 downsized_width = in->rect.width;
415 u32 downsized_height = in->rect.height;
416 u32 downsize_coeff_v = 0;
417 u32 downsize_coeff_h = 0;
418 u32 resized_width = out->rect.width;
419 u32 resized_height = out->rect.height;
420 u32 resize_coeff_h;
421 u32 resize_coeff_v;
422 u32 cols;
423 u32 rows;
424
425 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
426 resized_width = out->rect.height;
427 resized_height = out->rect.width;
428 }
429
430 /* Do not let invalid input lead to an endless loop below */
431 if (WARN_ON(resized_width == 0 || resized_height == 0))
432 return -EINVAL;
433
434 while (downsized_width >= resized_width * 2) {
435 downsized_width >>= 1;
436 downsize_coeff_h++;
437 }
438
439 while (downsized_height >= resized_height * 2) {
440 downsized_height >>= 1;
441 downsize_coeff_v++;
442 }
443
444 /*
445 * Calculate the bilinear resizing coefficients that could be used if
446 * we were converting with a single tile. The bottom right output pixel
447 * should sample as close as possible to the bottom right input pixel
448 * out of the decimator, but not overshoot it:
449 */
450 resize_coeff_h = 8192 * (downsized_width - 1) / (resized_width - 1);
451 resize_coeff_v = 8192 * (downsized_height - 1) / (resized_height - 1);
452
453 /*
454 * Both the output of the IC downsizing section before being passed to
455 * the IC main processing section and the final output of the IC main
456 * processing section must be <= 1024 pixels in both dimensions.
457 */
458 cols = num_stripes(max_t(u32, downsized_width, resized_width));
459 rows = num_stripes(max_t(u32, downsized_height, resized_height));
460
461 dev_dbg(ctx->chan->priv->ipu->dev,
462 "%s: hscale: >>%u, *8192/%u vscale: >>%u, *8192/%u, %ux%u tiles\n",
463 __func__, downsize_coeff_h, resize_coeff_h, downsize_coeff_v,
464 resize_coeff_v, cols, rows);
465
466 if (downsize_coeff_h > 2 || downsize_coeff_v > 2 ||
467 resize_coeff_h > 0x3fff || resize_coeff_v > 0x3fff)
468 return -EINVAL;
469
470 ctx->downsize_coeff_h = downsize_coeff_h;
471 ctx->downsize_coeff_v = downsize_coeff_v;
472 ctx->image_resize_coeff_h = resize_coeff_h;
473 ctx->image_resize_coeff_v = resize_coeff_v;
474 ctx->in.num_cols = cols;
475 ctx->in.num_rows = rows;
476
477 return 0;
478 }
479
480 #define round_closest(x, y) round_down((x) + (y)/2, (y))
481
482 /*
483 * Find the best aligned seam position for the given column / row index.
484 * Rotation and image offsets are out of scope.
485 *
486 * @index: column / row index, used to calculate valid interval
487 * @in_edge: input right / bottom edge
488 * @out_edge: output right / bottom edge
489 * @in_align: input alignment, either horizontal 8-byte line start address
490 * alignment, or pixel alignment due to image format
491 * @out_align: output alignment, either horizontal 8-byte line start address
492 * alignment, or pixel alignment due to image format or rotator
493 * block size
494 * @in_burst: horizontal input burst size in case of horizontal flip
495 * @out_burst: horizontal output burst size or rotator block size
496 * @downsize_coeff: downsizing section coefficient
497 * @resize_coeff: main processing section resizing coefficient
498 * @_in_seam: aligned input seam position return value
499 * @_out_seam: aligned output seam position return value
500 */
501 static void find_best_seam(struct ipu_image_convert_ctx *ctx,
502 unsigned int index,
503 unsigned int in_edge,
504 unsigned int out_edge,
505 unsigned int in_align,
506 unsigned int out_align,
507 unsigned int in_burst,
508 unsigned int out_burst,
509 unsigned int downsize_coeff,
510 unsigned int resize_coeff,
511 u32 *_in_seam,
512 u32 *_out_seam)
513 {
514 struct device *dev = ctx->chan->priv->ipu->dev;
515 unsigned int out_pos;
516 /* Input / output seam position candidates */
517 unsigned int out_seam = 0;
518 unsigned int in_seam = 0;
519 unsigned int min_diff = UINT_MAX;
520 unsigned int out_start;
521 unsigned int out_end;
522 unsigned int in_start;
523 unsigned int in_end;
524
525 /* Start within 1024 pixels of the right / bottom edge */
526 out_start = max_t(int, index * out_align, out_edge - 1024);
527 /* End before having to add more columns to the left / rows above */
528 out_end = min_t(unsigned int, out_edge, index * 1024 + 1);
529
530 /*
531 * Limit input seam position to make sure that the downsized input tile
532 * to the right or bottom does not exceed 1024 pixels.
533 */
534 in_start = max_t(int, index * in_align,
535 in_edge - (1024 << downsize_coeff));
536 in_end = min_t(unsigned int, in_edge,
537 index * (1024 << downsize_coeff) + 1);
538
539 /*
540 * Output tiles must start at a multiple of 8 bytes horizontally and
541 * possibly at an even line horizontally depending on the pixel format.
542 * Only consider output aligned positions for the seam.
543 */
544 out_start = round_up(out_start, out_align);
545 for (out_pos = out_start; out_pos < out_end; out_pos += out_align) {
546 unsigned int in_pos;
547 unsigned int in_pos_aligned;
548 unsigned int in_pos_rounded;
549 unsigned int diff;
550
551 /*
552 * Tiles in the right row / bottom column may not be allowed to
553 * overshoot horizontally / vertically. out_burst may be the
554 * actual DMA burst size, or the rotator block size.
555 */
556 if ((out_burst > 1) && (out_edge - out_pos) % out_burst)
557 continue;
558
559 /*
560 * Input sample position, corresponding to out_pos, 19.13 fixed
561 * point.
562 */
563 in_pos = (out_pos * resize_coeff) << downsize_coeff;
564 /*
565 * The closest input sample position that we could actually
566 * start the input tile at, 19.13 fixed point.
567 */
568 in_pos_aligned = round_closest(in_pos, 8192U * in_align);
569 /* Convert 19.13 fixed point to integer */
570 in_pos_rounded = in_pos_aligned / 8192U;
571
572 if (in_pos_rounded < in_start)
573 continue;
574 if (in_pos_rounded >= in_end)
575 break;
576
577 if ((in_burst > 1) &&
578 (in_edge - in_pos_rounded) % in_burst)
579 continue;
580
581 diff = abs_diff(in_pos, in_pos_aligned);
582 if (diff < min_diff) {
583 in_seam = in_pos_rounded;
584 out_seam = out_pos;
585 min_diff = diff;
586 }
587 }
588
589 *_out_seam = out_seam;
590 *_in_seam = in_seam;
591
592 dev_dbg(dev, "%s: out_seam %u(%u) in [%u, %u], in_seam %u(%u) in [%u, %u] diff %u.%03u\n",
593 __func__, out_seam, out_align, out_start, out_end,
594 in_seam, in_align, in_start, in_end, min_diff / 8192,
595 DIV_ROUND_CLOSEST(min_diff % 8192 * 1000, 8192));
596 }
597
598 /*
599 * Tile left edges are required to be aligned to multiples of 8 bytes
600 * by the IDMAC.
601 */
602 static inline u32 tile_left_align(const struct ipu_image_pixfmt *fmt)
603 {
604 if (fmt->planar)
605 return fmt->uv_packed ? 8 : 8 * fmt->uv_width_dec;
606 else
607 return fmt->bpp == 32 ? 2 : fmt->bpp == 16 ? 4 : 8;
608 }
609
610 /*
611 * Tile top edge alignment is only limited by chroma subsampling.
612 */
613 static inline u32 tile_top_align(const struct ipu_image_pixfmt *fmt)
614 {
615 return fmt->uv_height_dec > 1 ? 2 : 1;
616 }
617
618 static inline u32 tile_width_align(enum ipu_image_convert_type type,
619 const struct ipu_image_pixfmt *fmt,
620 enum ipu_rotate_mode rot_mode)
621 {
622 if (type == IMAGE_CONVERT_IN) {
623 /*
624 * The IC burst reads 8 pixels at a time. Reading beyond the
625 * end of the line is usually acceptable. Those pixels are
626 * ignored, unless the IC has to write the scaled line in
627 * reverse.
628 */
629 return (!ipu_rot_mode_is_irt(rot_mode) &&
630 (rot_mode & IPU_ROT_BIT_HFLIP)) ? 8 : 2;
631 }
632
633 /*
634 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
635 * formats to guarantee 8-byte aligned line start addresses in the
636 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
637 * for all other formats.
638 */
639 return (ipu_rot_mode_is_irt(rot_mode) &&
640 fmt->planar && !fmt->uv_packed) ?
641 8 * fmt->uv_width_dec : 8;
642 }
643
644 static inline u32 tile_height_align(enum ipu_image_convert_type type,
645 const struct ipu_image_pixfmt *fmt,
646 enum ipu_rotate_mode rot_mode)
647 {
648 if (type == IMAGE_CONVERT_IN || !ipu_rot_mode_is_irt(rot_mode))
649 return 2;
650
651 /*
652 * Align to 16x16 pixel blocks for planar 4:2:0 chroma subsampled
653 * formats to guarantee 8-byte aligned line start addresses in the
654 * chroma planes when IRT is used. Align to 8x8 pixel IRT block size
655 * for all other formats.
656 */
657 return (fmt->planar && !fmt->uv_packed) ? 8 * fmt->uv_width_dec : 8;
658 }
659
660 /*
661 * Fill in left position and width and for all tiles in an input column, and
662 * for all corresponding output tiles. If the 90° rotator is used, the output
663 * tiles are in a row, and output tile top position and height are set.
664 */
665 static void fill_tile_column(struct ipu_image_convert_ctx *ctx,
666 unsigned int col,
667 struct ipu_image_convert_image *in,
668 unsigned int in_left, unsigned int in_width,
669 struct ipu_image_convert_image *out,
670 unsigned int out_left, unsigned int out_width)
671 {
672 unsigned int row, tile_idx;
673 struct ipu_image_tile *in_tile, *out_tile;
674
675 for (row = 0; row < in->num_rows; row++) {
676 tile_idx = in->num_cols * row + col;
677 in_tile = &in->tile[tile_idx];
678 out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
679
680 in_tile->left = in_left;
681 in_tile->width = in_width;
682
683 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
684 out_tile->top = out_left;
685 out_tile->height = out_width;
686 } else {
687 out_tile->left = out_left;
688 out_tile->width = out_width;
689 }
690 }
691 }
692
693 /*
694 * Fill in top position and height and for all tiles in an input row, and
695 * for all corresponding output tiles. If the 90° rotator is used, the output
696 * tiles are in a column, and output tile left position and width are set.
697 */
698 static void fill_tile_row(struct ipu_image_convert_ctx *ctx, unsigned int row,
699 struct ipu_image_convert_image *in,
700 unsigned int in_top, unsigned int in_height,
701 struct ipu_image_convert_image *out,
702 unsigned int out_top, unsigned int out_height)
703 {
704 unsigned int col, tile_idx;
705 struct ipu_image_tile *in_tile, *out_tile;
706
707 for (col = 0; col < in->num_cols; col++) {
708 tile_idx = in->num_cols * row + col;
709 in_tile = &in->tile[tile_idx];
710 out_tile = &out->tile[ctx->out_tile_map[tile_idx]];
711
712 in_tile->top = in_top;
713 in_tile->height = in_height;
714
715 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
716 out_tile->left = out_top;
717 out_tile->width = out_height;
718 } else {
719 out_tile->top = out_top;
720 out_tile->height = out_height;
721 }
722 }
723 }
724
725 /*
726 * Find the best horizontal and vertical seam positions to split into tiles.
727 * Minimize the fractional part of the input sampling position for the
728 * top / left pixels of each tile.
729 */
730 static void find_seams(struct ipu_image_convert_ctx *ctx,
731 struct ipu_image_convert_image *in,
732 struct ipu_image_convert_image *out)
733 {
734 struct device *dev = ctx->chan->priv->ipu->dev;
735 unsigned int resized_width = out->base.rect.width;
736 unsigned int resized_height = out->base.rect.height;
737 unsigned int col;
738 unsigned int row;
739 unsigned int in_left_align = tile_left_align(in->fmt);
740 unsigned int in_top_align = tile_top_align(in->fmt);
741 unsigned int out_left_align = tile_left_align(out->fmt);
742 unsigned int out_top_align = tile_top_align(out->fmt);
743 unsigned int out_width_align = tile_width_align(out->type, out->fmt,
744 ctx->rot_mode);
745 unsigned int out_height_align = tile_height_align(out->type, out->fmt,
746 ctx->rot_mode);
747 unsigned int in_right = in->base.rect.width;
748 unsigned int in_bottom = in->base.rect.height;
749 unsigned int out_right = out->base.rect.width;
750 unsigned int out_bottom = out->base.rect.height;
751 unsigned int flipped_out_left;
752 unsigned int flipped_out_top;
753
754 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
755 /* Switch width/height and align top left to IRT block size */
756 resized_width = out->base.rect.height;
757 resized_height = out->base.rect.width;
758 out_left_align = out_height_align;
759 out_top_align = out_width_align;
760 out_width_align = out_left_align;
761 out_height_align = out_top_align;
762 out_right = out->base.rect.height;
763 out_bottom = out->base.rect.width;
764 }
765
766 for (col = in->num_cols - 1; col > 0; col--) {
767 bool allow_in_overshoot = ipu_rot_mode_is_irt(ctx->rot_mode) ||
768 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
769 bool allow_out_overshoot = (col < in->num_cols - 1) &&
770 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
771 unsigned int in_left;
772 unsigned int out_left;
773
774 /*
775 * Align input width to burst length if the scaling step flips
776 * horizontally.
777 */
778
779 find_best_seam(ctx, col,
780 in_right, out_right,
781 in_left_align, out_left_align,
782 allow_in_overshoot ? 1 : 8 /* burst length */,
783 allow_out_overshoot ? 1 : out_width_align,
784 ctx->downsize_coeff_h, ctx->image_resize_coeff_h,
785 &in_left, &out_left);
786
787 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
788 flipped_out_left = resized_width - out_right;
789 else
790 flipped_out_left = out_left;
791
792 fill_tile_column(ctx, col, in, in_left, in_right - in_left,
793 out, flipped_out_left, out_right - out_left);
794
795 dev_dbg(dev, "%s: col %u: %u, %u -> %u, %u\n", __func__, col,
796 in_left, in_right - in_left,
797 flipped_out_left, out_right - out_left);
798
799 in_right = in_left;
800 out_right = out_left;
801 }
802
803 flipped_out_left = (ctx->rot_mode & IPU_ROT_BIT_HFLIP) ?
804 resized_width - out_right : 0;
805
806 fill_tile_column(ctx, 0, in, 0, in_right,
807 out, flipped_out_left, out_right);
808
809 dev_dbg(dev, "%s: col 0: 0, %u -> %u, %u\n", __func__,
810 in_right, flipped_out_left, out_right);
811
812 for (row = in->num_rows - 1; row > 0; row--) {
813 bool allow_overshoot = row < in->num_rows - 1;
814 unsigned int in_top;
815 unsigned int out_top;
816
817 find_best_seam(ctx, row,
818 in_bottom, out_bottom,
819 in_top_align, out_top_align,
820 1, allow_overshoot ? 1 : out_height_align,
821 ctx->downsize_coeff_v, ctx->image_resize_coeff_v,
822 &in_top, &out_top);
823
824 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
825 ipu_rot_mode_is_irt(ctx->rot_mode))
826 flipped_out_top = resized_height - out_bottom;
827 else
828 flipped_out_top = out_top;
829
830 fill_tile_row(ctx, row, in, in_top, in_bottom - in_top,
831 out, flipped_out_top, out_bottom - out_top);
832
833 dev_dbg(dev, "%s: row %u: %u, %u -> %u, %u\n", __func__, row,
834 in_top, in_bottom - in_top,
835 flipped_out_top, out_bottom - out_top);
836
837 in_bottom = in_top;
838 out_bottom = out_top;
839 }
840
841 if ((ctx->rot_mode & IPU_ROT_BIT_VFLIP) ^
842 ipu_rot_mode_is_irt(ctx->rot_mode))
843 flipped_out_top = resized_height - out_bottom;
844 else
845 flipped_out_top = 0;
846
847 fill_tile_row(ctx, 0, in, 0, in_bottom,
848 out, flipped_out_top, out_bottom);
849
850 dev_dbg(dev, "%s: row 0: 0, %u -> %u, %u\n", __func__,
851 in_bottom, flipped_out_top, out_bottom);
852 }
853
854 static int calc_tile_dimensions(struct ipu_image_convert_ctx *ctx,
855 struct ipu_image_convert_image *image)
856 {
857 struct ipu_image_convert_chan *chan = ctx->chan;
858 struct ipu_image_convert_priv *priv = chan->priv;
859 unsigned int max_width = 1024;
860 unsigned int max_height = 1024;
861 unsigned int i;
862
863 if (image->type == IMAGE_CONVERT_IN) {
864 /* Up to 4096x4096 input tile size */
865 max_width <<= ctx->downsize_coeff_h;
866 max_height <<= ctx->downsize_coeff_v;
867 }
868
869 for (i = 0; i < ctx->num_tiles; i++) {
870 struct ipu_image_tile *tile;
871 const unsigned int row = i / image->num_cols;
872 const unsigned int col = i % image->num_cols;
873
874 if (image->type == IMAGE_CONVERT_OUT)
875 tile = &image->tile[ctx->out_tile_map[i]];
876 else
877 tile = &image->tile[i];
878
879 tile->size = ((tile->height * image->fmt->bpp) >> 3) *
880 tile->width;
881
882 if (image->fmt->planar) {
883 tile->stride = tile->width;
884 tile->rot_stride = tile->height;
885 } else {
886 tile->stride =
887 (image->fmt->bpp * tile->width) >> 3;
888 tile->rot_stride =
889 (image->fmt->bpp * tile->height) >> 3;
890 }
891
892 dev_dbg(priv->ipu->dev,
893 "task %u: ctx %p: %s@[%u,%u]: %ux%u@%u,%u\n",
894 chan->ic_task, ctx,
895 image->type == IMAGE_CONVERT_IN ? "Input" : "Output",
896 row, col,
897 tile->width, tile->height, tile->left, tile->top);
898
899 if (!tile->width || tile->width > max_width ||
900 !tile->height || tile->height > max_height) {
901 dev_err(priv->ipu->dev, "invalid %s tile size: %ux%u\n",
902 image->type == IMAGE_CONVERT_IN ? "input" :
903 "output", tile->width, tile->height);
904 return -EINVAL;
905 }
906 }
907
908 return 0;
909 }
910
911 /*
912 * Use the rotation transformation to find the tile coordinates
913 * (row, col) of a tile in the destination frame that corresponds
914 * to the given tile coordinates of a source frame. The destination
915 * coordinate is then converted to a tile index.
916 */
917 static int transform_tile_index(struct ipu_image_convert_ctx *ctx,
918 int src_row, int src_col)
919 {
920 struct ipu_image_convert_chan *chan = ctx->chan;
921 struct ipu_image_convert_priv *priv = chan->priv;
922 struct ipu_image_convert_image *s_image = &ctx->in;
923 struct ipu_image_convert_image *d_image = &ctx->out;
924 int dst_row, dst_col;
925
926 /* with no rotation it's a 1:1 mapping */
927 if (ctx->rot_mode == IPU_ROTATE_NONE)
928 return src_row * s_image->num_cols + src_col;
929
930 /*
931 * before doing the transform, first we have to translate
932 * source row,col for an origin in the center of s_image
933 */
934 src_row = src_row * 2 - (s_image->num_rows - 1);
935 src_col = src_col * 2 - (s_image->num_cols - 1);
936
937 /* do the rotation transform */
938 if (ctx->rot_mode & IPU_ROT_BIT_90) {
939 dst_col = -src_row;
940 dst_row = src_col;
941 } else {
942 dst_col = src_col;
943 dst_row = src_row;
944 }
945
946 /* apply flip */
947 if (ctx->rot_mode & IPU_ROT_BIT_HFLIP)
948 dst_col = -dst_col;
949 if (ctx->rot_mode & IPU_ROT_BIT_VFLIP)
950 dst_row = -dst_row;
951
952 dev_dbg(priv->ipu->dev, "task %u: ctx %p: [%d,%d] --> [%d,%d]\n",
953 chan->ic_task, ctx, src_col, src_row, dst_col, dst_row);
954
955 /*
956 * finally translate dest row,col using an origin in upper
957 * left of d_image
958 */
959 dst_row += d_image->num_rows - 1;
960 dst_col += d_image->num_cols - 1;
961 dst_row /= 2;
962 dst_col /= 2;
963
964 return dst_row * d_image->num_cols + dst_col;
965 }
966
967 /*
968 * Fill the out_tile_map[] with transformed destination tile indeces.
969 */
970 static void calc_out_tile_map(struct ipu_image_convert_ctx *ctx)
971 {
972 struct ipu_image_convert_image *s_image = &ctx->in;
973 unsigned int row, col, tile = 0;
974
975 for (row = 0; row < s_image->num_rows; row++) {
976 for (col = 0; col < s_image->num_cols; col++) {
977 ctx->out_tile_map[tile] =
978 transform_tile_index(ctx, row, col);
979 tile++;
980 }
981 }
982 }
983
984 static int calc_tile_offsets_planar(struct ipu_image_convert_ctx *ctx,
985 struct ipu_image_convert_image *image)
986 {
987 struct ipu_image_convert_chan *chan = ctx->chan;
988 struct ipu_image_convert_priv *priv = chan->priv;
989 const struct ipu_image_pixfmt *fmt = image->fmt;
990 unsigned int row, col, tile = 0;
991 u32 H, top, y_stride, uv_stride;
992 u32 uv_row_off, uv_col_off, uv_off, u_off, v_off;
993 u32 y_row_off, y_col_off, y_off;
994 u32 y_size, uv_size;
995
996 /* setup some convenience vars */
997 H = image->base.pix.height;
998
999 y_stride = image->stride;
1000 uv_stride = y_stride / fmt->uv_width_dec;
1001 if (fmt->uv_packed)
1002 uv_stride *= 2;
1003
1004 y_size = H * y_stride;
1005 uv_size = y_size / (fmt->uv_width_dec * fmt->uv_height_dec);
1006
1007 for (row = 0; row < image->num_rows; row++) {
1008 top = image->tile[tile].top;
1009 y_row_off = top * y_stride;
1010 uv_row_off = (top * uv_stride) / fmt->uv_height_dec;
1011
1012 for (col = 0; col < image->num_cols; col++) {
1013 y_col_off = image->tile[tile].left;
1014 uv_col_off = y_col_off / fmt->uv_width_dec;
1015 if (fmt->uv_packed)
1016 uv_col_off *= 2;
1017
1018 y_off = y_row_off + y_col_off;
1019 uv_off = uv_row_off + uv_col_off;
1020
1021 u_off = y_size - y_off + uv_off;
1022 v_off = (fmt->uv_packed) ? 0 : u_off + uv_size;
1023 if (fmt->uv_swapped)
1024 swap(u_off, v_off);
1025
1026 image->tile[tile].offset = y_off;
1027 image->tile[tile].u_off = u_off;
1028 image->tile[tile++].v_off = v_off;
1029
1030 if ((y_off & 0x7) || (u_off & 0x7) || (v_off & 0x7)) {
1031 dev_err(priv->ipu->dev,
1032 "task %u: ctx %p: %s@[%d,%d]: "
1033 "y_off %08x, u_off %08x, v_off %08x\n",
1034 chan->ic_task, ctx,
1035 image->type == IMAGE_CONVERT_IN ?
1036 "Input" : "Output", row, col,
1037 y_off, u_off, v_off);
1038 return -EINVAL;
1039 }
1040 }
1041 }
1042
1043 return 0;
1044 }
1045
1046 static int calc_tile_offsets_packed(struct ipu_image_convert_ctx *ctx,
1047 struct ipu_image_convert_image *image)
1048 {
1049 struct ipu_image_convert_chan *chan = ctx->chan;
1050 struct ipu_image_convert_priv *priv = chan->priv;
1051 const struct ipu_image_pixfmt *fmt = image->fmt;
1052 unsigned int row, col, tile = 0;
1053 u32 bpp, stride, offset;
1054 u32 row_off, col_off;
1055
1056 /* setup some convenience vars */
1057 stride = image->stride;
1058 bpp = fmt->bpp;
1059
1060 for (row = 0; row < image->num_rows; row++) {
1061 row_off = image->tile[tile].top * stride;
1062
1063 for (col = 0; col < image->num_cols; col++) {
1064 col_off = (image->tile[tile].left * bpp) >> 3;
1065
1066 offset = row_off + col_off;
1067
1068 image->tile[tile].offset = offset;
1069 image->tile[tile].u_off = 0;
1070 image->tile[tile++].v_off = 0;
1071
1072 if (offset & 0x7) {
1073 dev_err(priv->ipu->dev,
1074 "task %u: ctx %p: %s@[%d,%d]: "
1075 "phys %08x\n",
1076 chan->ic_task, ctx,
1077 image->type == IMAGE_CONVERT_IN ?
1078 "Input" : "Output", row, col,
1079 row_off + col_off);
1080 return -EINVAL;
1081 }
1082 }
1083 }
1084
1085 return 0;
1086 }
1087
1088 static int calc_tile_offsets(struct ipu_image_convert_ctx *ctx,
1089 struct ipu_image_convert_image *image)
1090 {
1091 if (image->fmt->planar)
1092 return calc_tile_offsets_planar(ctx, image);
1093
1094 return calc_tile_offsets_packed(ctx, image);
1095 }
1096
1097 /*
1098 * Calculate the resizing ratio for the IC main processing section given input
1099 * size, fixed downsizing coefficient, and output size.
1100 * Either round to closest for the next tile's first pixel to minimize seams
1101 * and distortion (for all but right column / bottom row), or round down to
1102 * avoid sampling beyond the edges of the input image for this tile's last
1103 * pixel.
1104 * Returns the resizing coefficient, resizing ratio is 8192.0 / resize_coeff.
1105 */
1106 static u32 calc_resize_coeff(u32 input_size, u32 downsize_coeff,
1107 u32 output_size, bool allow_overshoot)
1108 {
1109 u32 downsized = input_size >> downsize_coeff;
1110
1111 if (allow_overshoot)
1112 return DIV_ROUND_CLOSEST(8192 * downsized, output_size);
1113 else
1114 return 8192 * (downsized - 1) / (output_size - 1);
1115 }
1116
1117 /*
1118 * Slightly modify resize coefficients per tile to hide the bilinear
1119 * interpolator reset at tile borders, shifting the right / bottom edge
1120 * by up to a half input pixel. This removes noticeable seams between
1121 * tiles at higher upscaling factors.
1122 */
1123 static void calc_tile_resize_coefficients(struct ipu_image_convert_ctx *ctx)
1124 {
1125 struct ipu_image_convert_chan *chan = ctx->chan;
1126 struct ipu_image_convert_priv *priv = chan->priv;
1127 struct ipu_image_tile *in_tile, *out_tile;
1128 unsigned int col, row, tile_idx;
1129 unsigned int last_output;
1130
1131 for (col = 0; col < ctx->in.num_cols; col++) {
1132 bool closest = (col < ctx->in.num_cols - 1) &&
1133 !(ctx->rot_mode & IPU_ROT_BIT_HFLIP);
1134 u32 resized_width;
1135 u32 resize_coeff_h;
1136 u32 in_width;
1137
1138 tile_idx = col;
1139 in_tile = &ctx->in.tile[tile_idx];
1140 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1141
1142 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1143 resized_width = out_tile->height;
1144 else
1145 resized_width = out_tile->width;
1146
1147 resize_coeff_h = calc_resize_coeff(in_tile->width,
1148 ctx->downsize_coeff_h,
1149 resized_width, closest);
1150
1151 dev_dbg(priv->ipu->dev, "%s: column %u hscale: *8192/%u\n",
1152 __func__, col, resize_coeff_h);
1153
1154 /*
1155 * With the horizontal scaling factor known, round up resized
1156 * width (output width or height) to burst size.
1157 */
1158 resized_width = round_up(resized_width, 8);
1159
1160 /*
1161 * Calculate input width from the last accessed input pixel
1162 * given resized width and scaling coefficients. Round up to
1163 * burst size.
1164 */
1165 last_output = resized_width - 1;
1166 if (closest && ((last_output * resize_coeff_h) % 8192))
1167 last_output++;
1168 in_width = round_up(
1169 (DIV_ROUND_UP(last_output * resize_coeff_h, 8192) + 1)
1170 << ctx->downsize_coeff_h, 8);
1171
1172 for (row = 0; row < ctx->in.num_rows; row++) {
1173 tile_idx = row * ctx->in.num_cols + col;
1174 in_tile = &ctx->in.tile[tile_idx];
1175 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1176
1177 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1178 out_tile->height = resized_width;
1179 else
1180 out_tile->width = resized_width;
1181
1182 in_tile->width = in_width;
1183 }
1184
1185 ctx->resize_coeffs_h[col] = resize_coeff_h;
1186 }
1187
1188 for (row = 0; row < ctx->in.num_rows; row++) {
1189 bool closest = (row < ctx->in.num_rows - 1) &&
1190 !(ctx->rot_mode & IPU_ROT_BIT_VFLIP);
1191 u32 resized_height;
1192 u32 resize_coeff_v;
1193 u32 in_height;
1194
1195 tile_idx = row * ctx->in.num_cols;
1196 in_tile = &ctx->in.tile[tile_idx];
1197 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1198
1199 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1200 resized_height = out_tile->width;
1201 else
1202 resized_height = out_tile->height;
1203
1204 resize_coeff_v = calc_resize_coeff(in_tile->height,
1205 ctx->downsize_coeff_v,
1206 resized_height, closest);
1207
1208 dev_dbg(priv->ipu->dev, "%s: row %u vscale: *8192/%u\n",
1209 __func__, row, resize_coeff_v);
1210
1211 /*
1212 * With the vertical scaling factor known, round up resized
1213 * height (output width or height) to IDMAC limitations.
1214 */
1215 resized_height = round_up(resized_height, 2);
1216
1217 /*
1218 * Calculate input width from the last accessed input pixel
1219 * given resized height and scaling coefficients. Align to
1220 * IDMAC restrictions.
1221 */
1222 last_output = resized_height - 1;
1223 if (closest && ((last_output * resize_coeff_v) % 8192))
1224 last_output++;
1225 in_height = round_up(
1226 (DIV_ROUND_UP(last_output * resize_coeff_v, 8192) + 1)
1227 << ctx->downsize_coeff_v, 2);
1228
1229 for (col = 0; col < ctx->in.num_cols; col++) {
1230 tile_idx = row * ctx->in.num_cols + col;
1231 in_tile = &ctx->in.tile[tile_idx];
1232 out_tile = &ctx->out.tile[ctx->out_tile_map[tile_idx]];
1233
1234 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1235 out_tile->width = resized_height;
1236 else
1237 out_tile->height = resized_height;
1238
1239 in_tile->height = in_height;
1240 }
1241
1242 ctx->resize_coeffs_v[row] = resize_coeff_v;
1243 }
1244 }
1245
1246 /*
1247 * return the number of runs in given queue (pending_q or done_q)
1248 * for this context. hold irqlock when calling.
1249 */
1250 static int get_run_count(struct ipu_image_convert_ctx *ctx,
1251 struct list_head *q)
1252 {
1253 struct ipu_image_convert_run *run;
1254 int count = 0;
1255
1256 lockdep_assert_held(&ctx->chan->irqlock);
1257
1258 list_for_each_entry(run, q, list) {
1259 if (run->ctx == ctx)
1260 count++;
1261 }
1262
1263 return count;
1264 }
1265
1266 static void convert_stop(struct ipu_image_convert_run *run)
1267 {
1268 struct ipu_image_convert_ctx *ctx = run->ctx;
1269 struct ipu_image_convert_chan *chan = ctx->chan;
1270 struct ipu_image_convert_priv *priv = chan->priv;
1271
1272 dev_dbg(priv->ipu->dev, "%s: task %u: stopping ctx %p run %p\n",
1273 __func__, chan->ic_task, ctx, run);
1274
1275 /* disable IC tasks and the channels */
1276 ipu_ic_task_disable(chan->ic);
1277 ipu_idmac_disable_channel(chan->in_chan);
1278 ipu_idmac_disable_channel(chan->out_chan);
1279
1280 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1281 ipu_idmac_disable_channel(chan->rotation_in_chan);
1282 ipu_idmac_disable_channel(chan->rotation_out_chan);
1283 ipu_idmac_unlink(chan->out_chan, chan->rotation_in_chan);
1284 }
1285
1286 ipu_ic_disable(chan->ic);
1287 }
1288
1289 static void init_idmac_channel(struct ipu_image_convert_ctx *ctx,
1290 struct ipuv3_channel *channel,
1291 struct ipu_image_convert_image *image,
1292 enum ipu_rotate_mode rot_mode,
1293 bool rot_swap_width_height,
1294 unsigned int tile)
1295 {
1296 struct ipu_image_convert_chan *chan = ctx->chan;
1297 unsigned int burst_size;
1298 u32 width, height, stride;
1299 dma_addr_t addr0, addr1 = 0;
1300 struct ipu_image tile_image;
1301 unsigned int tile_idx[2];
1302
1303 if (image->type == IMAGE_CONVERT_OUT) {
1304 tile_idx[0] = ctx->out_tile_map[tile];
1305 tile_idx[1] = ctx->out_tile_map[1];
1306 } else {
1307 tile_idx[0] = tile;
1308 tile_idx[1] = 1;
1309 }
1310
1311 if (rot_swap_width_height) {
1312 width = image->tile[tile_idx[0]].height;
1313 height = image->tile[tile_idx[0]].width;
1314 stride = image->tile[tile_idx[0]].rot_stride;
1315 addr0 = ctx->rot_intermediate[0].phys;
1316 if (ctx->double_buffering)
1317 addr1 = ctx->rot_intermediate[1].phys;
1318 } else {
1319 width = image->tile[tile_idx[0]].width;
1320 height = image->tile[tile_idx[0]].height;
1321 stride = image->stride;
1322 addr0 = image->base.phys0 +
1323 image->tile[tile_idx[0]].offset;
1324 if (ctx->double_buffering)
1325 addr1 = image->base.phys0 +
1326 image->tile[tile_idx[1]].offset;
1327 }
1328
1329 ipu_cpmem_zero(channel);
1330
1331 memset(&tile_image, 0, sizeof(tile_image));
1332 tile_image.pix.width = tile_image.rect.width = width;
1333 tile_image.pix.height = tile_image.rect.height = height;
1334 tile_image.pix.bytesperline = stride;
1335 tile_image.pix.pixelformat = image->fmt->fourcc;
1336 tile_image.phys0 = addr0;
1337 tile_image.phys1 = addr1;
1338 if (image->fmt->planar && !rot_swap_width_height) {
1339 tile_image.u_offset = image->tile[tile_idx[0]].u_off;
1340 tile_image.v_offset = image->tile[tile_idx[0]].v_off;
1341 }
1342
1343 ipu_cpmem_set_image(channel, &tile_image);
1344
1345 if (rot_mode)
1346 ipu_cpmem_set_rotation(channel, rot_mode);
1347
1348 /*
1349 * Skip writing U and V components to odd rows in the output
1350 * channels for planar 4:2:0.
1351 */
1352 if ((channel == chan->out_chan ||
1353 channel == chan->rotation_out_chan) &&
1354 image->fmt->planar && image->fmt->uv_height_dec == 2)
1355 ipu_cpmem_skip_odd_chroma_rows(channel);
1356
1357 if (channel == chan->rotation_in_chan ||
1358 channel == chan->rotation_out_chan) {
1359 burst_size = 8;
1360 ipu_cpmem_set_block_mode(channel);
1361 } else
1362 burst_size = (width % 16) ? 8 : 16;
1363
1364 ipu_cpmem_set_burstsize(channel, burst_size);
1365
1366 ipu_ic_task_idma_init(chan->ic, channel, width, height,
1367 burst_size, rot_mode);
1368
1369 /*
1370 * Setting a non-zero AXI ID collides with the PRG AXI snooping, so
1371 * only do this when there is no PRG present.
1372 */
1373 if (!channel->ipu->prg_priv)
1374 ipu_cpmem_set_axi_id(channel, 1);
1375
1376 ipu_idmac_set_double_buffer(channel, ctx->double_buffering);
1377 }
1378
1379 static int convert_start(struct ipu_image_convert_run *run, unsigned int tile)
1380 {
1381 struct ipu_image_convert_ctx *ctx = run->ctx;
1382 struct ipu_image_convert_chan *chan = ctx->chan;
1383 struct ipu_image_convert_priv *priv = chan->priv;
1384 struct ipu_image_convert_image *s_image = &ctx->in;
1385 struct ipu_image_convert_image *d_image = &ctx->out;
1386 unsigned int dst_tile = ctx->out_tile_map[tile];
1387 unsigned int dest_width, dest_height;
1388 unsigned int col, row;
1389 u32 rsc;
1390 int ret;
1391
1392 dev_dbg(priv->ipu->dev, "%s: task %u: starting ctx %p run %p tile %u -> %u\n",
1393 __func__, chan->ic_task, ctx, run, tile, dst_tile);
1394
1395 /* clear EOF irq mask */
1396 ctx->eof_mask = 0;
1397
1398 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1399 /* swap width/height for resizer */
1400 dest_width = d_image->tile[dst_tile].height;
1401 dest_height = d_image->tile[dst_tile].width;
1402 } else {
1403 dest_width = d_image->tile[dst_tile].width;
1404 dest_height = d_image->tile[dst_tile].height;
1405 }
1406
1407 row = tile / s_image->num_cols;
1408 col = tile % s_image->num_cols;
1409
1410 rsc = (ctx->downsize_coeff_v << 30) |
1411 (ctx->resize_coeffs_v[row] << 16) |
1412 (ctx->downsize_coeff_h << 14) |
1413 (ctx->resize_coeffs_h[col]);
1414
1415 dev_dbg(priv->ipu->dev, "%s: %ux%u -> %ux%u (rsc = 0x%x)\n",
1416 __func__, s_image->tile[tile].width,
1417 s_image->tile[tile].height, dest_width, dest_height, rsc);
1418
1419 /* setup the IC resizer and CSC */
1420 ret = ipu_ic_task_init_rsc(chan->ic, &ctx->csc,
1421 s_image->tile[tile].width,
1422 s_image->tile[tile].height,
1423 dest_width,
1424 dest_height,
1425 rsc);
1426 if (ret) {
1427 dev_err(priv->ipu->dev, "ipu_ic_task_init failed, %d\n", ret);
1428 return ret;
1429 }
1430
1431 /* init the source MEM-->IC PP IDMAC channel */
1432 init_idmac_channel(ctx, chan->in_chan, s_image,
1433 IPU_ROTATE_NONE, false, tile);
1434
1435 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1436 /* init the IC PP-->MEM IDMAC channel */
1437 init_idmac_channel(ctx, chan->out_chan, d_image,
1438 IPU_ROTATE_NONE, true, tile);
1439
1440 /* init the MEM-->IC PP ROT IDMAC channel */
1441 init_idmac_channel(ctx, chan->rotation_in_chan, d_image,
1442 ctx->rot_mode, true, tile);
1443
1444 /* init the destination IC PP ROT-->MEM IDMAC channel */
1445 init_idmac_channel(ctx, chan->rotation_out_chan, d_image,
1446 IPU_ROTATE_NONE, false, tile);
1447
1448 /* now link IC PP-->MEM to MEM-->IC PP ROT */
1449 ipu_idmac_link(chan->out_chan, chan->rotation_in_chan);
1450 } else {
1451 /* init the destination IC PP-->MEM IDMAC channel */
1452 init_idmac_channel(ctx, chan->out_chan, d_image,
1453 ctx->rot_mode, false, tile);
1454 }
1455
1456 /* enable the IC */
1457 ipu_ic_enable(chan->ic);
1458
1459 /* set buffers ready */
1460 ipu_idmac_select_buffer(chan->in_chan, 0);
1461 ipu_idmac_select_buffer(chan->out_chan, 0);
1462 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1463 ipu_idmac_select_buffer(chan->rotation_out_chan, 0);
1464 if (ctx->double_buffering) {
1465 ipu_idmac_select_buffer(chan->in_chan, 1);
1466 ipu_idmac_select_buffer(chan->out_chan, 1);
1467 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1468 ipu_idmac_select_buffer(chan->rotation_out_chan, 1);
1469 }
1470
1471 /* enable the channels! */
1472 ipu_idmac_enable_channel(chan->in_chan);
1473 ipu_idmac_enable_channel(chan->out_chan);
1474 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1475 ipu_idmac_enable_channel(chan->rotation_in_chan);
1476 ipu_idmac_enable_channel(chan->rotation_out_chan);
1477 }
1478
1479 ipu_ic_task_enable(chan->ic);
1480
1481 ipu_cpmem_dump(chan->in_chan);
1482 ipu_cpmem_dump(chan->out_chan);
1483 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
1484 ipu_cpmem_dump(chan->rotation_in_chan);
1485 ipu_cpmem_dump(chan->rotation_out_chan);
1486 }
1487
1488 ipu_dump(priv->ipu);
1489
1490 return 0;
1491 }
1492
1493 /* hold irqlock when calling */
1494 static int do_run(struct ipu_image_convert_run *run)
1495 {
1496 struct ipu_image_convert_ctx *ctx = run->ctx;
1497 struct ipu_image_convert_chan *chan = ctx->chan;
1498
1499 lockdep_assert_held(&chan->irqlock);
1500
1501 ctx->in.base.phys0 = run->in_phys;
1502 ctx->out.base.phys0 = run->out_phys;
1503
1504 ctx->cur_buf_num = 0;
1505 ctx->next_tile = 1;
1506
1507 /* remove run from pending_q and set as current */
1508 list_del(&run->list);
1509 chan->current_run = run;
1510
1511 return convert_start(run, 0);
1512 }
1513
1514 /* hold irqlock when calling */
1515 static void run_next(struct ipu_image_convert_chan *chan)
1516 {
1517 struct ipu_image_convert_priv *priv = chan->priv;
1518 struct ipu_image_convert_run *run, *tmp;
1519 int ret;
1520
1521 lockdep_assert_held(&chan->irqlock);
1522
1523 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
1524 /* skip contexts that are aborting */
1525 if (run->ctx->aborting) {
1526 dev_dbg(priv->ipu->dev,
1527 "%s: task %u: skipping aborting ctx %p run %p\n",
1528 __func__, chan->ic_task, run->ctx, run);
1529 continue;
1530 }
1531
1532 ret = do_run(run);
1533 if (!ret)
1534 break;
1535
1536 /*
1537 * something went wrong with start, add the run
1538 * to done q and continue to the next run in the
1539 * pending q.
1540 */
1541 run->status = ret;
1542 list_add_tail(&run->list, &chan->done_q);
1543 chan->current_run = NULL;
1544 }
1545 }
1546
1547 static void empty_done_q(struct ipu_image_convert_chan *chan)
1548 {
1549 struct ipu_image_convert_priv *priv = chan->priv;
1550 struct ipu_image_convert_run *run;
1551 unsigned long flags;
1552
1553 spin_lock_irqsave(&chan->irqlock, flags);
1554
1555 while (!list_empty(&chan->done_q)) {
1556 run = list_entry(chan->done_q.next,
1557 struct ipu_image_convert_run,
1558 list);
1559
1560 list_del(&run->list);
1561
1562 dev_dbg(priv->ipu->dev,
1563 "%s: task %u: completing ctx %p run %p with %d\n",
1564 __func__, chan->ic_task, run->ctx, run, run->status);
1565
1566 /* call the completion callback and free the run */
1567 spin_unlock_irqrestore(&chan->irqlock, flags);
1568 run->ctx->complete(run, run->ctx->complete_context);
1569 spin_lock_irqsave(&chan->irqlock, flags);
1570 }
1571
1572 spin_unlock_irqrestore(&chan->irqlock, flags);
1573 }
1574
1575 /*
1576 * the bottom half thread clears out the done_q, calling the
1577 * completion handler for each.
1578 */
1579 static irqreturn_t do_bh(int irq, void *dev_id)
1580 {
1581 struct ipu_image_convert_chan *chan = dev_id;
1582 struct ipu_image_convert_priv *priv = chan->priv;
1583 struct ipu_image_convert_ctx *ctx;
1584 unsigned long flags;
1585
1586 dev_dbg(priv->ipu->dev, "%s: task %u: enter\n", __func__,
1587 chan->ic_task);
1588
1589 empty_done_q(chan);
1590
1591 spin_lock_irqsave(&chan->irqlock, flags);
1592
1593 /*
1594 * the done_q is cleared out, signal any contexts
1595 * that are aborting that abort can complete.
1596 */
1597 list_for_each_entry(ctx, &chan->ctx_list, list) {
1598 if (ctx->aborting) {
1599 dev_dbg(priv->ipu->dev,
1600 "%s: task %u: signaling abort for ctx %p\n",
1601 __func__, chan->ic_task, ctx);
1602 complete_all(&ctx->aborted);
1603 }
1604 }
1605
1606 spin_unlock_irqrestore(&chan->irqlock, flags);
1607
1608 dev_dbg(priv->ipu->dev, "%s: task %u: exit\n", __func__,
1609 chan->ic_task);
1610
1611 return IRQ_HANDLED;
1612 }
1613
1614 static bool ic_settings_changed(struct ipu_image_convert_ctx *ctx)
1615 {
1616 unsigned int cur_tile = ctx->next_tile - 1;
1617 unsigned int next_tile = ctx->next_tile;
1618
1619 if (ctx->resize_coeffs_h[cur_tile % ctx->in.num_cols] !=
1620 ctx->resize_coeffs_h[next_tile % ctx->in.num_cols] ||
1621 ctx->resize_coeffs_v[cur_tile / ctx->in.num_cols] !=
1622 ctx->resize_coeffs_v[next_tile / ctx->in.num_cols] ||
1623 ctx->in.tile[cur_tile].width != ctx->in.tile[next_tile].width ||
1624 ctx->in.tile[cur_tile].height != ctx->in.tile[next_tile].height ||
1625 ctx->out.tile[cur_tile].width != ctx->out.tile[next_tile].width ||
1626 ctx->out.tile[cur_tile].height != ctx->out.tile[next_tile].height)
1627 return true;
1628
1629 return false;
1630 }
1631
1632 /* hold irqlock when calling */
1633 static irqreturn_t do_tile_complete(struct ipu_image_convert_run *run)
1634 {
1635 struct ipu_image_convert_ctx *ctx = run->ctx;
1636 struct ipu_image_convert_chan *chan = ctx->chan;
1637 struct ipu_image_tile *src_tile, *dst_tile;
1638 struct ipu_image_convert_image *s_image = &ctx->in;
1639 struct ipu_image_convert_image *d_image = &ctx->out;
1640 struct ipuv3_channel *outch;
1641 unsigned int dst_idx;
1642
1643 lockdep_assert_held(&chan->irqlock);
1644
1645 outch = ipu_rot_mode_is_irt(ctx->rot_mode) ?
1646 chan->rotation_out_chan : chan->out_chan;
1647
1648 /*
1649 * It is difficult to stop the channel DMA before the channels
1650 * enter the paused state. Without double-buffering the channels
1651 * are always in a paused state when the EOF irq occurs, so it
1652 * is safe to stop the channels now. For double-buffering we
1653 * just ignore the abort until the operation completes, when it
1654 * is safe to shut down.
1655 */
1656 if (ctx->aborting && !ctx->double_buffering) {
1657 convert_stop(run);
1658 run->status = -EIO;
1659 goto done;
1660 }
1661
1662 if (ctx->next_tile == ctx->num_tiles) {
1663 /*
1664 * the conversion is complete
1665 */
1666 convert_stop(run);
1667 run->status = 0;
1668 goto done;
1669 }
1670
1671 /*
1672 * not done, place the next tile buffers.
1673 */
1674 if (!ctx->double_buffering) {
1675 if (ic_settings_changed(ctx)) {
1676 convert_stop(run);
1677 convert_start(run, ctx->next_tile);
1678 } else {
1679 src_tile = &s_image->tile[ctx->next_tile];
1680 dst_idx = ctx->out_tile_map[ctx->next_tile];
1681 dst_tile = &d_image->tile[dst_idx];
1682
1683 ipu_cpmem_set_buffer(chan->in_chan, 0,
1684 s_image->base.phys0 +
1685 src_tile->offset);
1686 ipu_cpmem_set_buffer(outch, 0,
1687 d_image->base.phys0 +
1688 dst_tile->offset);
1689 if (s_image->fmt->planar)
1690 ipu_cpmem_set_uv_offset(chan->in_chan,
1691 src_tile->u_off,
1692 src_tile->v_off);
1693 if (d_image->fmt->planar)
1694 ipu_cpmem_set_uv_offset(outch,
1695 dst_tile->u_off,
1696 dst_tile->v_off);
1697
1698 ipu_idmac_select_buffer(chan->in_chan, 0);
1699 ipu_idmac_select_buffer(outch, 0);
1700 }
1701 } else if (ctx->next_tile < ctx->num_tiles - 1) {
1702
1703 src_tile = &s_image->tile[ctx->next_tile + 1];
1704 dst_idx = ctx->out_tile_map[ctx->next_tile + 1];
1705 dst_tile = &d_image->tile[dst_idx];
1706
1707 ipu_cpmem_set_buffer(chan->in_chan, ctx->cur_buf_num,
1708 s_image->base.phys0 + src_tile->offset);
1709 ipu_cpmem_set_buffer(outch, ctx->cur_buf_num,
1710 d_image->base.phys0 + dst_tile->offset);
1711
1712 ipu_idmac_select_buffer(chan->in_chan, ctx->cur_buf_num);
1713 ipu_idmac_select_buffer(outch, ctx->cur_buf_num);
1714
1715 ctx->cur_buf_num ^= 1;
1716 }
1717
1718 ctx->eof_mask = 0; /* clear EOF irq mask for next tile */
1719 ctx->next_tile++;
1720 return IRQ_HANDLED;
1721 done:
1722 list_add_tail(&run->list, &chan->done_q);
1723 chan->current_run = NULL;
1724 run_next(chan);
1725 return IRQ_WAKE_THREAD;
1726 }
1727
1728 static irqreturn_t eof_irq(int irq, void *data)
1729 {
1730 struct ipu_image_convert_chan *chan = data;
1731 struct ipu_image_convert_priv *priv = chan->priv;
1732 struct ipu_image_convert_ctx *ctx;
1733 struct ipu_image_convert_run *run;
1734 irqreturn_t ret = IRQ_HANDLED;
1735 bool tile_complete = false;
1736 unsigned long flags;
1737
1738 spin_lock_irqsave(&chan->irqlock, flags);
1739
1740 /* get current run and its context */
1741 run = chan->current_run;
1742 if (!run) {
1743 ret = IRQ_NONE;
1744 goto out;
1745 }
1746
1747 ctx = run->ctx;
1748
1749 if (irq == chan->in_eof_irq) {
1750 ctx->eof_mask |= EOF_IRQ_IN;
1751 } else if (irq == chan->out_eof_irq) {
1752 ctx->eof_mask |= EOF_IRQ_OUT;
1753 } else if (irq == chan->rot_in_eof_irq ||
1754 irq == chan->rot_out_eof_irq) {
1755 if (!ipu_rot_mode_is_irt(ctx->rot_mode)) {
1756 /* this was NOT a rotation op, shouldn't happen */
1757 dev_err(priv->ipu->dev,
1758 "Unexpected rotation interrupt\n");
1759 goto out;
1760 }
1761 ctx->eof_mask |= (irq == chan->rot_in_eof_irq) ?
1762 EOF_IRQ_ROT_IN : EOF_IRQ_ROT_OUT;
1763 } else {
1764 dev_err(priv->ipu->dev, "Received unknown irq %d\n", irq);
1765 ret = IRQ_NONE;
1766 goto out;
1767 }
1768
1769 if (ipu_rot_mode_is_irt(ctx->rot_mode))
1770 tile_complete = (ctx->eof_mask == EOF_IRQ_ROT_COMPLETE);
1771 else
1772 tile_complete = (ctx->eof_mask == EOF_IRQ_COMPLETE);
1773
1774 if (tile_complete)
1775 ret = do_tile_complete(run);
1776 out:
1777 spin_unlock_irqrestore(&chan->irqlock, flags);
1778 return ret;
1779 }
1780
1781 /*
1782 * try to force the completion of runs for this ctx. Called when
1783 * abort wait times out in ipu_image_convert_abort().
1784 */
1785 static void force_abort(struct ipu_image_convert_ctx *ctx)
1786 {
1787 struct ipu_image_convert_chan *chan = ctx->chan;
1788 struct ipu_image_convert_run *run;
1789 unsigned long flags;
1790
1791 spin_lock_irqsave(&chan->irqlock, flags);
1792
1793 run = chan->current_run;
1794 if (run && run->ctx == ctx) {
1795 convert_stop(run);
1796 run->status = -EIO;
1797 list_add_tail(&run->list, &chan->done_q);
1798 chan->current_run = NULL;
1799 run_next(chan);
1800 }
1801
1802 spin_unlock_irqrestore(&chan->irqlock, flags);
1803
1804 empty_done_q(chan);
1805 }
1806
1807 static void release_ipu_resources(struct ipu_image_convert_chan *chan)
1808 {
1809 if (chan->in_eof_irq >= 0)
1810 free_irq(chan->in_eof_irq, chan);
1811 if (chan->rot_in_eof_irq >= 0)
1812 free_irq(chan->rot_in_eof_irq, chan);
1813 if (chan->out_eof_irq >= 0)
1814 free_irq(chan->out_eof_irq, chan);
1815 if (chan->rot_out_eof_irq >= 0)
1816 free_irq(chan->rot_out_eof_irq, chan);
1817
1818 if (!IS_ERR_OR_NULL(chan->in_chan))
1819 ipu_idmac_put(chan->in_chan);
1820 if (!IS_ERR_OR_NULL(chan->out_chan))
1821 ipu_idmac_put(chan->out_chan);
1822 if (!IS_ERR_OR_NULL(chan->rotation_in_chan))
1823 ipu_idmac_put(chan->rotation_in_chan);
1824 if (!IS_ERR_OR_NULL(chan->rotation_out_chan))
1825 ipu_idmac_put(chan->rotation_out_chan);
1826 if (!IS_ERR_OR_NULL(chan->ic))
1827 ipu_ic_put(chan->ic);
1828
1829 chan->in_chan = chan->out_chan = chan->rotation_in_chan =
1830 chan->rotation_out_chan = NULL;
1831 chan->in_eof_irq = -1;
1832 chan->rot_in_eof_irq = -1;
1833 chan->out_eof_irq = -1;
1834 chan->rot_out_eof_irq = -1;
1835 }
1836
1837 static int get_eof_irq(struct ipu_image_convert_chan *chan,
1838 struct ipuv3_channel *channel)
1839 {
1840 struct ipu_image_convert_priv *priv = chan->priv;
1841 int ret, irq;
1842
1843 irq = ipu_idmac_channel_irq(priv->ipu, channel, IPU_IRQ_EOF);
1844
1845 ret = request_threaded_irq(irq, eof_irq, do_bh, 0, "ipu-ic", chan);
1846 if (ret < 0) {
1847 dev_err(priv->ipu->dev, "could not acquire irq %d\n", irq);
1848 return ret;
1849 }
1850
1851 return irq;
1852 }
1853
1854 static int get_ipu_resources(struct ipu_image_convert_chan *chan)
1855 {
1856 const struct ipu_image_convert_dma_chan *dma = chan->dma_ch;
1857 struct ipu_image_convert_priv *priv = chan->priv;
1858 int ret;
1859
1860 /* get IC */
1861 chan->ic = ipu_ic_get(priv->ipu, chan->ic_task);
1862 if (IS_ERR(chan->ic)) {
1863 dev_err(priv->ipu->dev, "could not acquire IC\n");
1864 ret = PTR_ERR(chan->ic);
1865 goto err;
1866 }
1867
1868 /* get IDMAC channels */
1869 chan->in_chan = ipu_idmac_get(priv->ipu, dma->in);
1870 chan->out_chan = ipu_idmac_get(priv->ipu, dma->out);
1871 if (IS_ERR(chan->in_chan) || IS_ERR(chan->out_chan)) {
1872 dev_err(priv->ipu->dev, "could not acquire idmac channels\n");
1873 ret = -EBUSY;
1874 goto err;
1875 }
1876
1877 chan->rotation_in_chan = ipu_idmac_get(priv->ipu, dma->rot_in);
1878 chan->rotation_out_chan = ipu_idmac_get(priv->ipu, dma->rot_out);
1879 if (IS_ERR(chan->rotation_in_chan) || IS_ERR(chan->rotation_out_chan)) {
1880 dev_err(priv->ipu->dev,
1881 "could not acquire idmac rotation channels\n");
1882 ret = -EBUSY;
1883 goto err;
1884 }
1885
1886 /* acquire the EOF interrupts */
1887 ret = get_eof_irq(chan, chan->in_chan);
1888 if (ret < 0) {
1889 chan->in_eof_irq = -1;
1890 goto err;
1891 }
1892 chan->in_eof_irq = ret;
1893
1894 ret = get_eof_irq(chan, chan->rotation_in_chan);
1895 if (ret < 0) {
1896 chan->rot_in_eof_irq = -1;
1897 goto err;
1898 }
1899 chan->rot_in_eof_irq = ret;
1900
1901 ret = get_eof_irq(chan, chan->out_chan);
1902 if (ret < 0) {
1903 chan->out_eof_irq = -1;
1904 goto err;
1905 }
1906 chan->out_eof_irq = ret;
1907
1908 ret = get_eof_irq(chan, chan->rotation_out_chan);
1909 if (ret < 0) {
1910 chan->rot_out_eof_irq = -1;
1911 goto err;
1912 }
1913 chan->rot_out_eof_irq = ret;
1914
1915 return 0;
1916 err:
1917 release_ipu_resources(chan);
1918 return ret;
1919 }
1920
1921 static int fill_image(struct ipu_image_convert_ctx *ctx,
1922 struct ipu_image_convert_image *ic_image,
1923 struct ipu_image *image,
1924 enum ipu_image_convert_type type)
1925 {
1926 struct ipu_image_convert_priv *priv = ctx->chan->priv;
1927
1928 ic_image->base = *image;
1929 ic_image->type = type;
1930
1931 ic_image->fmt = get_format(image->pix.pixelformat);
1932 if (!ic_image->fmt) {
1933 dev_err(priv->ipu->dev, "pixelformat not supported for %s\n",
1934 type == IMAGE_CONVERT_OUT ? "Output" : "Input");
1935 return -EINVAL;
1936 }
1937
1938 if (ic_image->fmt->planar)
1939 ic_image->stride = ic_image->base.pix.width;
1940 else
1941 ic_image->stride = ic_image->base.pix.bytesperline;
1942
1943 return 0;
1944 }
1945
1946 /* borrowed from drivers/media/v4l2-core/v4l2-common.c */
1947 static unsigned int clamp_align(unsigned int x, unsigned int min,
1948 unsigned int max, unsigned int align)
1949 {
1950 /* Bits that must be zero to be aligned */
1951 unsigned int mask = ~((1 << align) - 1);
1952
1953 /* Clamp to aligned min and max */
1954 x = clamp(x, (min + ~mask) & mask, max & mask);
1955
1956 /* Round to nearest aligned value */
1957 if (align)
1958 x = (x + (1 << (align - 1))) & mask;
1959
1960 return x;
1961 }
1962
1963 /* Adjusts input/output images to IPU restrictions */
1964 void ipu_image_convert_adjust(struct ipu_image *in, struct ipu_image *out,
1965 enum ipu_rotate_mode rot_mode)
1966 {
1967 const struct ipu_image_pixfmt *infmt, *outfmt;
1968 u32 w_align_out, h_align_out;
1969 u32 w_align_in, h_align_in;
1970
1971 infmt = get_format(in->pix.pixelformat);
1972 outfmt = get_format(out->pix.pixelformat);
1973
1974 /* set some default pixel formats if needed */
1975 if (!infmt) {
1976 in->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1977 infmt = get_format(V4L2_PIX_FMT_RGB24);
1978 }
1979 if (!outfmt) {
1980 out->pix.pixelformat = V4L2_PIX_FMT_RGB24;
1981 outfmt = get_format(V4L2_PIX_FMT_RGB24);
1982 }
1983
1984 /* image converter does not handle fields */
1985 in->pix.field = out->pix.field = V4L2_FIELD_NONE;
1986
1987 /* resizer cannot downsize more than 4:1 */
1988 if (ipu_rot_mode_is_irt(rot_mode)) {
1989 out->pix.height = max_t(__u32, out->pix.height,
1990 in->pix.width / 4);
1991 out->pix.width = max_t(__u32, out->pix.width,
1992 in->pix.height / 4);
1993 } else {
1994 out->pix.width = max_t(__u32, out->pix.width,
1995 in->pix.width / 4);
1996 out->pix.height = max_t(__u32, out->pix.height,
1997 in->pix.height / 4);
1998 }
1999
2000 /* align input width/height */
2001 w_align_in = ilog2(tile_width_align(IMAGE_CONVERT_IN, infmt,
2002 rot_mode));
2003 h_align_in = ilog2(tile_height_align(IMAGE_CONVERT_IN, infmt,
2004 rot_mode));
2005 in->pix.width = clamp_align(in->pix.width, MIN_W, MAX_W,
2006 w_align_in);
2007 in->pix.height = clamp_align(in->pix.height, MIN_H, MAX_H,
2008 h_align_in);
2009
2010 /* align output width/height */
2011 w_align_out = ilog2(tile_width_align(IMAGE_CONVERT_OUT, outfmt,
2012 rot_mode));
2013 h_align_out = ilog2(tile_height_align(IMAGE_CONVERT_OUT, outfmt,
2014 rot_mode));
2015 out->pix.width = clamp_align(out->pix.width, MIN_W, MAX_W,
2016 w_align_out);
2017 out->pix.height = clamp_align(out->pix.height, MIN_H, MAX_H,
2018 h_align_out);
2019
2020 /* set input/output strides and image sizes */
2021 in->pix.bytesperline = infmt->planar ?
2022 clamp_align(in->pix.width, 2 << w_align_in, MAX_W,
2023 w_align_in) :
2024 clamp_align((in->pix.width * infmt->bpp) >> 3,
2025 ((2 << w_align_in) * infmt->bpp) >> 3,
2026 (MAX_W * infmt->bpp) >> 3,
2027 w_align_in);
2028 in->pix.sizeimage = infmt->planar ?
2029 (in->pix.height * in->pix.bytesperline * infmt->bpp) >> 3 :
2030 in->pix.height * in->pix.bytesperline;
2031 out->pix.bytesperline = outfmt->planar ? out->pix.width :
2032 (out->pix.width * outfmt->bpp) >> 3;
2033 out->pix.sizeimage = outfmt->planar ?
2034 (out->pix.height * out->pix.bytesperline * outfmt->bpp) >> 3 :
2035 out->pix.height * out->pix.bytesperline;
2036 }
2037 EXPORT_SYMBOL_GPL(ipu_image_convert_adjust);
2038
2039 /*
2040 * this is used by ipu_image_convert_prepare() to verify set input and
2041 * output images are valid before starting the conversion. Clients can
2042 * also call it before calling ipu_image_convert_prepare().
2043 */
2044 int ipu_image_convert_verify(struct ipu_image *in, struct ipu_image *out,
2045 enum ipu_rotate_mode rot_mode)
2046 {
2047 struct ipu_image testin, testout;
2048
2049 testin = *in;
2050 testout = *out;
2051
2052 ipu_image_convert_adjust(&testin, &testout, rot_mode);
2053
2054 if (testin.pix.width != in->pix.width ||
2055 testin.pix.height != in->pix.height ||
2056 testout.pix.width != out->pix.width ||
2057 testout.pix.height != out->pix.height)
2058 return -EINVAL;
2059
2060 return 0;
2061 }
2062 EXPORT_SYMBOL_GPL(ipu_image_convert_verify);
2063
2064 /*
2065 * Call ipu_image_convert_prepare() to prepare for the conversion of
2066 * given images and rotation mode. Returns a new conversion context.
2067 */
2068 struct ipu_image_convert_ctx *
2069 ipu_image_convert_prepare(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2070 struct ipu_image *in, struct ipu_image *out,
2071 enum ipu_rotate_mode rot_mode,
2072 ipu_image_convert_cb_t complete,
2073 void *complete_context)
2074 {
2075 struct ipu_image_convert_priv *priv = ipu->image_convert_priv;
2076 struct ipu_image_convert_image *s_image, *d_image;
2077 struct ipu_image_convert_chan *chan;
2078 struct ipu_image_convert_ctx *ctx;
2079 unsigned long flags;
2080 unsigned int i;
2081 bool get_res;
2082 int ret;
2083
2084 if (!in || !out || !complete ||
2085 (ic_task != IC_TASK_VIEWFINDER &&
2086 ic_task != IC_TASK_POST_PROCESSOR))
2087 return ERR_PTR(-EINVAL);
2088
2089 /* verify the in/out images before continuing */
2090 ret = ipu_image_convert_verify(in, out, rot_mode);
2091 if (ret) {
2092 dev_err(priv->ipu->dev, "%s: in/out formats invalid\n",
2093 __func__);
2094 return ERR_PTR(ret);
2095 }
2096
2097 chan = &priv->chan[ic_task];
2098
2099 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
2100 if (!ctx)
2101 return ERR_PTR(-ENOMEM);
2102
2103 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p\n", __func__,
2104 chan->ic_task, ctx);
2105
2106 ctx->chan = chan;
2107 init_completion(&ctx->aborted);
2108
2109 ctx->rot_mode = rot_mode;
2110
2111 /* Sets ctx->in.num_rows/cols as well */
2112 ret = calc_image_resize_coefficients(ctx, in, out);
2113 if (ret)
2114 goto out_free;
2115
2116 s_image = &ctx->in;
2117 d_image = &ctx->out;
2118
2119 /* set tiling and rotation */
2120 if (ipu_rot_mode_is_irt(rot_mode)) {
2121 d_image->num_rows = s_image->num_cols;
2122 d_image->num_cols = s_image->num_rows;
2123 } else {
2124 d_image->num_rows = s_image->num_rows;
2125 d_image->num_cols = s_image->num_cols;
2126 }
2127
2128 ctx->num_tiles = d_image->num_cols * d_image->num_rows;
2129
2130 ret = fill_image(ctx, s_image, in, IMAGE_CONVERT_IN);
2131 if (ret)
2132 goto out_free;
2133 ret = fill_image(ctx, d_image, out, IMAGE_CONVERT_OUT);
2134 if (ret)
2135 goto out_free;
2136
2137 calc_out_tile_map(ctx);
2138
2139 find_seams(ctx, s_image, d_image);
2140
2141 ret = calc_tile_dimensions(ctx, s_image);
2142 if (ret)
2143 goto out_free;
2144
2145 ret = calc_tile_offsets(ctx, s_image);
2146 if (ret)
2147 goto out_free;
2148
2149 calc_tile_dimensions(ctx, d_image);
2150 ret = calc_tile_offsets(ctx, d_image);
2151 if (ret)
2152 goto out_free;
2153
2154 calc_tile_resize_coefficients(ctx);
2155
2156 ret = ipu_ic_calc_csc(&ctx->csc,
2157 s_image->base.pix.ycbcr_enc,
2158 s_image->base.pix.quantization,
2159 ipu_pixelformat_to_colorspace(s_image->fmt->fourcc),
2160 d_image->base.pix.ycbcr_enc,
2161 d_image->base.pix.quantization,
2162 ipu_pixelformat_to_colorspace(d_image->fmt->fourcc));
2163 if (ret)
2164 goto out_free;
2165
2166 dump_format(ctx, s_image);
2167 dump_format(ctx, d_image);
2168
2169 ctx->complete = complete;
2170 ctx->complete_context = complete_context;
2171
2172 /*
2173 * Can we use double-buffering for this operation? If there is
2174 * only one tile (the whole image can be converted in a single
2175 * operation) there's no point in using double-buffering. Also,
2176 * the IPU's IDMAC channels allow only a single U and V plane
2177 * offset shared between both buffers, but these offsets change
2178 * for every tile, and therefore would have to be updated for
2179 * each buffer which is not possible. So double-buffering is
2180 * impossible when either the source or destination images are
2181 * a planar format (YUV420, YUV422P, etc.). Further, differently
2182 * sized tiles or different resizing coefficients per tile
2183 * prevent double-buffering as well.
2184 */
2185 ctx->double_buffering = (ctx->num_tiles > 1 &&
2186 !s_image->fmt->planar &&
2187 !d_image->fmt->planar);
2188 for (i = 1; i < ctx->num_tiles; i++) {
2189 if (ctx->in.tile[i].width != ctx->in.tile[0].width ||
2190 ctx->in.tile[i].height != ctx->in.tile[0].height ||
2191 ctx->out.tile[i].width != ctx->out.tile[0].width ||
2192 ctx->out.tile[i].height != ctx->out.tile[0].height) {
2193 ctx->double_buffering = false;
2194 break;
2195 }
2196 }
2197 for (i = 1; i < ctx->in.num_cols; i++) {
2198 if (ctx->resize_coeffs_h[i] != ctx->resize_coeffs_h[0]) {
2199 ctx->double_buffering = false;
2200 break;
2201 }
2202 }
2203 for (i = 1; i < ctx->in.num_rows; i++) {
2204 if (ctx->resize_coeffs_v[i] != ctx->resize_coeffs_v[0]) {
2205 ctx->double_buffering = false;
2206 break;
2207 }
2208 }
2209
2210 if (ipu_rot_mode_is_irt(ctx->rot_mode)) {
2211 unsigned long intermediate_size = d_image->tile[0].size;
2212
2213 for (i = 1; i < ctx->num_tiles; i++) {
2214 if (d_image->tile[i].size > intermediate_size)
2215 intermediate_size = d_image->tile[i].size;
2216 }
2217
2218 ret = alloc_dma_buf(priv, &ctx->rot_intermediate[0],
2219 intermediate_size);
2220 if (ret)
2221 goto out_free;
2222 if (ctx->double_buffering) {
2223 ret = alloc_dma_buf(priv,
2224 &ctx->rot_intermediate[1],
2225 intermediate_size);
2226 if (ret)
2227 goto out_free_dmabuf0;
2228 }
2229 }
2230
2231 spin_lock_irqsave(&chan->irqlock, flags);
2232
2233 get_res = list_empty(&chan->ctx_list);
2234
2235 list_add_tail(&ctx->list, &chan->ctx_list);
2236
2237 spin_unlock_irqrestore(&chan->irqlock, flags);
2238
2239 if (get_res) {
2240 ret = get_ipu_resources(chan);
2241 if (ret)
2242 goto out_free_dmabuf1;
2243 }
2244
2245 return ctx;
2246
2247 out_free_dmabuf1:
2248 free_dma_buf(priv, &ctx->rot_intermediate[1]);
2249 spin_lock_irqsave(&chan->irqlock, flags);
2250 list_del(&ctx->list);
2251 spin_unlock_irqrestore(&chan->irqlock, flags);
2252 out_free_dmabuf0:
2253 free_dma_buf(priv, &ctx->rot_intermediate[0]);
2254 out_free:
2255 kfree(ctx);
2256 return ERR_PTR(ret);
2257 }
2258 EXPORT_SYMBOL_GPL(ipu_image_convert_prepare);
2259
2260 /*
2261 * Carry out a single image conversion run. Only the physaddr's of the input
2262 * and output image buffers are needed. The conversion context must have
2263 * been created previously with ipu_image_convert_prepare().
2264 */
2265 int ipu_image_convert_queue(struct ipu_image_convert_run *run)
2266 {
2267 struct ipu_image_convert_chan *chan;
2268 struct ipu_image_convert_priv *priv;
2269 struct ipu_image_convert_ctx *ctx;
2270 unsigned long flags;
2271 int ret = 0;
2272
2273 if (!run || !run->ctx || !run->in_phys || !run->out_phys)
2274 return -EINVAL;
2275
2276 ctx = run->ctx;
2277 chan = ctx->chan;
2278 priv = chan->priv;
2279
2280 dev_dbg(priv->ipu->dev, "%s: task %u: ctx %p run %p\n", __func__,
2281 chan->ic_task, ctx, run);
2282
2283 INIT_LIST_HEAD(&run->list);
2284
2285 spin_lock_irqsave(&chan->irqlock, flags);
2286
2287 if (ctx->aborting) {
2288 ret = -EIO;
2289 goto unlock;
2290 }
2291
2292 list_add_tail(&run->list, &chan->pending_q);
2293
2294 if (!chan->current_run) {
2295 ret = do_run(run);
2296 if (ret)
2297 chan->current_run = NULL;
2298 }
2299 unlock:
2300 spin_unlock_irqrestore(&chan->irqlock, flags);
2301 return ret;
2302 }
2303 EXPORT_SYMBOL_GPL(ipu_image_convert_queue);
2304
2305 /* Abort any active or pending conversions for this context */
2306 static void __ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2307 {
2308 struct ipu_image_convert_chan *chan = ctx->chan;
2309 struct ipu_image_convert_priv *priv = chan->priv;
2310 struct ipu_image_convert_run *run, *active_run, *tmp;
2311 unsigned long flags;
2312 int run_count, ret;
2313
2314 spin_lock_irqsave(&chan->irqlock, flags);
2315
2316 /* move all remaining pending runs in this context to done_q */
2317 list_for_each_entry_safe(run, tmp, &chan->pending_q, list) {
2318 if (run->ctx != ctx)
2319 continue;
2320 run->status = -EIO;
2321 list_move_tail(&run->list, &chan->done_q);
2322 }
2323
2324 run_count = get_run_count(ctx, &chan->done_q);
2325 active_run = (chan->current_run && chan->current_run->ctx == ctx) ?
2326 chan->current_run : NULL;
2327
2328 if (active_run)
2329 reinit_completion(&ctx->aborted);
2330
2331 ctx->aborting = true;
2332
2333 spin_unlock_irqrestore(&chan->irqlock, flags);
2334
2335 if (!run_count && !active_run) {
2336 dev_dbg(priv->ipu->dev,
2337 "%s: task %u: no abort needed for ctx %p\n",
2338 __func__, chan->ic_task, ctx);
2339 return;
2340 }
2341
2342 if (!active_run) {
2343 empty_done_q(chan);
2344 return;
2345 }
2346
2347 dev_dbg(priv->ipu->dev,
2348 "%s: task %u: wait for completion: %d runs\n",
2349 __func__, chan->ic_task, run_count);
2350
2351 ret = wait_for_completion_timeout(&ctx->aborted,
2352 msecs_to_jiffies(10000));
2353 if (ret == 0) {
2354 dev_warn(priv->ipu->dev, "%s: timeout\n", __func__);
2355 force_abort(ctx);
2356 }
2357 }
2358
2359 void ipu_image_convert_abort(struct ipu_image_convert_ctx *ctx)
2360 {
2361 __ipu_image_convert_abort(ctx);
2362 ctx->aborting = false;
2363 }
2364 EXPORT_SYMBOL_GPL(ipu_image_convert_abort);
2365
2366 /* Unprepare image conversion context */
2367 void ipu_image_convert_unprepare(struct ipu_image_convert_ctx *ctx)
2368 {
2369 struct ipu_image_convert_chan *chan = ctx->chan;
2370 struct ipu_image_convert_priv *priv = chan->priv;
2371 unsigned long flags;
2372 bool put_res;
2373
2374 /* make sure no runs are hanging around */
2375 __ipu_image_convert_abort(ctx);
2376
2377 dev_dbg(priv->ipu->dev, "%s: task %u: removing ctx %p\n", __func__,
2378 chan->ic_task, ctx);
2379
2380 spin_lock_irqsave(&chan->irqlock, flags);
2381
2382 list_del(&ctx->list);
2383
2384 put_res = list_empty(&chan->ctx_list);
2385
2386 spin_unlock_irqrestore(&chan->irqlock, flags);
2387
2388 if (put_res)
2389 release_ipu_resources(chan);
2390
2391 free_dma_buf(priv, &ctx->rot_intermediate[1]);
2392 free_dma_buf(priv, &ctx->rot_intermediate[0]);
2393
2394 kfree(ctx);
2395 }
2396 EXPORT_SYMBOL_GPL(ipu_image_convert_unprepare);
2397
2398 /*
2399 * "Canned" asynchronous single image conversion. Allocates and returns
2400 * a new conversion run. On successful return the caller must free the
2401 * run and call ipu_image_convert_unprepare() after conversion completes.
2402 */
2403 struct ipu_image_convert_run *
2404 ipu_image_convert(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2405 struct ipu_image *in, struct ipu_image *out,
2406 enum ipu_rotate_mode rot_mode,
2407 ipu_image_convert_cb_t complete,
2408 void *complete_context)
2409 {
2410 struct ipu_image_convert_ctx *ctx;
2411 struct ipu_image_convert_run *run;
2412 int ret;
2413
2414 ctx = ipu_image_convert_prepare(ipu, ic_task, in, out, rot_mode,
2415 complete, complete_context);
2416 if (IS_ERR(ctx))
2417 return ERR_CAST(ctx);
2418
2419 run = kzalloc(sizeof(*run), GFP_KERNEL);
2420 if (!run) {
2421 ipu_image_convert_unprepare(ctx);
2422 return ERR_PTR(-ENOMEM);
2423 }
2424
2425 run->ctx = ctx;
2426 run->in_phys = in->phys0;
2427 run->out_phys = out->phys0;
2428
2429 ret = ipu_image_convert_queue(run);
2430 if (ret) {
2431 ipu_image_convert_unprepare(ctx);
2432 kfree(run);
2433 return ERR_PTR(ret);
2434 }
2435
2436 return run;
2437 }
2438 EXPORT_SYMBOL_GPL(ipu_image_convert);
2439
2440 /* "Canned" synchronous single image conversion */
2441 static void image_convert_sync_complete(struct ipu_image_convert_run *run,
2442 void *data)
2443 {
2444 struct completion *comp = data;
2445
2446 complete(comp);
2447 }
2448
2449 int ipu_image_convert_sync(struct ipu_soc *ipu, enum ipu_ic_task ic_task,
2450 struct ipu_image *in, struct ipu_image *out,
2451 enum ipu_rotate_mode rot_mode)
2452 {
2453 struct ipu_image_convert_run *run;
2454 struct completion comp;
2455 int ret;
2456
2457 init_completion(&comp);
2458
2459 run = ipu_image_convert(ipu, ic_task, in, out, rot_mode,
2460 image_convert_sync_complete, &comp);
2461 if (IS_ERR(run))
2462 return PTR_ERR(run);
2463
2464 ret = wait_for_completion_timeout(&comp, msecs_to_jiffies(10000));
2465 ret = (ret == 0) ? -ETIMEDOUT : 0;
2466
2467 ipu_image_convert_unprepare(run->ctx);
2468 kfree(run);
2469
2470 return ret;
2471 }
2472 EXPORT_SYMBOL_GPL(ipu_image_convert_sync);
2473
2474 int ipu_image_convert_init(struct ipu_soc *ipu, struct device *dev)
2475 {
2476 struct ipu_image_convert_priv *priv;
2477 int i;
2478
2479 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
2480 if (!priv)
2481 return -ENOMEM;
2482
2483 ipu->image_convert_priv = priv;
2484 priv->ipu = ipu;
2485
2486 for (i = 0; i < IC_NUM_TASKS; i++) {
2487 struct ipu_image_convert_chan *chan = &priv->chan[i];
2488
2489 chan->ic_task = i;
2490 chan->priv = priv;
2491 chan->dma_ch = &image_convert_dma_chan[i];
2492 chan->in_eof_irq = -1;
2493 chan->rot_in_eof_irq = -1;
2494 chan->out_eof_irq = -1;
2495 chan->rot_out_eof_irq = -1;
2496
2497 spin_lock_init(&chan->irqlock);
2498 INIT_LIST_HEAD(&chan->ctx_list);
2499 INIT_LIST_HEAD(&chan->pending_q);
2500 INIT_LIST_HEAD(&chan->done_q);
2501 }
2502
2503 return 0;
2504 }
2505
2506 void ipu_image_convert_exit(struct ipu_soc *ipu)
2507 {
2508 }
Kernel DRM miscellaneous fixes and cross-tree changes