1 /* SPDX-License-Identifier: GPL-2.0 */
3 * Hantro VPU codec driver
5 * Copyright 2018 Google LLC.
6 * Tomasz Figa <tfiga@chromium.org>
12 #include <linux/interrupt.h>
13 #include <linux/v4l2-controls.h>
14 #include <media/v4l2-ctrls.h>
15 #include <media/v4l2-vp9.h>
16 #include <media/videobuf2-core.h>
18 #include "rockchip_av1_entropymode.h"
19 #include "rockchip_av1_filmgrain.h"
21 #define DEC_8190_ALIGN_MASK 0x07U
25 #define MB_WIDTH(w) DIV_ROUND_UP(w, MB_DIM)
26 #define MB_HEIGHT(h) DIV_ROUND_UP(h, MB_DIM)
28 #define FMT_MIN_WIDTH 48
29 #define FMT_MIN_HEIGHT 48
30 #define FMT_HD_WIDTH 1280
31 #define FMT_HD_HEIGHT 720
32 #define FMT_FHD_WIDTH 1920
33 #define FMT_FHD_HEIGHT 1088
34 #define FMT_UHD_WIDTH 3840
35 #define FMT_UHD_HEIGHT 2160
36 #define FMT_4K_WIDTH 4096
37 #define FMT_4K_HEIGHT 2304
39 #define NUM_REF_PICTURES (V4L2_HEVC_DPB_ENTRIES_NUM_MAX + 1)
41 #define AV1_MAX_FRAME_BUF_COUNT (V4L2_AV1_TOTAL_REFS_PER_FRAME + 1)
43 #define MAX_POSTPROC_BUFFERS 64
45 #define CBS_SIZE 16 /* compression table size in bytes */
46 #define CBS_LUMA 8 /* luminance CBS is composed of 1 8x8 coded block */
47 #define CBS_CHROMA_W (8 * 2) /* chrominance CBS is composed of two 8x4 coded
48 * blocks, with Cb CB first then Cr CB following
50 #define CBS_CHROMA_H 4
55 struct hantro_variant
;
58 * struct hantro_aux_buf - auxiliary DMA buffer for hardware data
60 * @cpu: CPU pointer to the buffer.
61 * @dma: DMA address of the buffer.
62 * @size: Size of the buffer.
63 * @attrs: Attributes of the DMA mapping.
65 struct hantro_aux_buf
{
72 /* Max. number of entries in the DPB (HW limitation). */
73 #define HANTRO_H264_DPB_SIZE 16
76 * struct hantro_h264_dec_ctrls
78 * @decode: Decode params
79 * @scaling: Scaling info
83 struct hantro_h264_dec_ctrls
{
84 const struct v4l2_ctrl_h264_decode_params
*decode
;
85 const struct v4l2_ctrl_h264_scaling_matrix
*scaling
;
86 const struct v4l2_ctrl_h264_sps
*sps
;
87 const struct v4l2_ctrl_h264_pps
*pps
;
91 * struct hantro_h264_dec_reflists
97 struct hantro_h264_dec_reflists
{
98 struct v4l2_h264_reference p
[V4L2_H264_REF_LIST_LEN
];
99 struct v4l2_h264_reference b0
[V4L2_H264_REF_LIST_LEN
];
100 struct v4l2_h264_reference b1
[V4L2_H264_REF_LIST_LEN
];
104 * struct hantro_h264_dec_hw_ctx
106 * @priv: Private auxiliary buffer for hardware.
108 * @reflists: P/B0/B1 reflists
109 * @ctrls: V4L2 controls attached to a run
110 * @dpb_longterm: DPB long-term
111 * @dpb_valid: DPB valid
112 * @cur_poc: Current picture order count
114 struct hantro_h264_dec_hw_ctx
{
115 struct hantro_aux_buf priv
;
116 struct v4l2_h264_dpb_entry dpb
[HANTRO_H264_DPB_SIZE
];
117 struct hantro_h264_dec_reflists reflists
;
118 struct hantro_h264_dec_ctrls ctrls
;
125 * struct hantro_hevc_dec_ctrls
126 * @decode_params: Decode params
127 * @scaling: Scaling matrix
130 * @hevc_hdr_skip_length: the number of data (in bits) to skip in the
131 * slice segment header syntax after 'slice type'
134 struct hantro_hevc_dec_ctrls
{
135 const struct v4l2_ctrl_hevc_decode_params
*decode_params
;
136 const struct v4l2_ctrl_hevc_scaling_matrix
*scaling
;
137 const struct v4l2_ctrl_hevc_sps
*sps
;
138 const struct v4l2_ctrl_hevc_pps
*pps
;
139 u32 hevc_hdr_skip_length
;
143 * struct hantro_hevc_dec_hw_ctx
144 * @tile_sizes: Tile sizes buffer
145 * @tile_filter: Tile vertical filter buffer
146 * @tile_sao: Tile SAO buffer
147 * @tile_bsd: Tile BSD control buffer
148 * @ref_bufs: Internal reference buffers
149 * @scaling_lists: Scaling lists buffer
150 * @ref_bufs_poc: Internal reference buffers picture order count
151 * @ref_bufs_used: Bitfield of used reference buffers
152 * @ctrls: V4L2 controls attached to a run
153 * @num_tile_cols_allocated: number of allocated tiles
154 * @use_compression: use reference buffer compression
156 struct hantro_hevc_dec_hw_ctx
{
157 struct hantro_aux_buf tile_sizes
;
158 struct hantro_aux_buf tile_filter
;
159 struct hantro_aux_buf tile_sao
;
160 struct hantro_aux_buf tile_bsd
;
161 struct hantro_aux_buf ref_bufs
[NUM_REF_PICTURES
];
162 struct hantro_aux_buf scaling_lists
;
163 s32 ref_bufs_poc
[NUM_REF_PICTURES
];
165 struct hantro_hevc_dec_ctrls ctrls
;
166 unsigned int num_tile_cols_allocated
;
167 bool use_compression
;
171 * struct hantro_mpeg2_dec_hw_ctx
173 * @qtable: Quantization table
175 struct hantro_mpeg2_dec_hw_ctx
{
176 struct hantro_aux_buf qtable
;
180 * struct hantro_vp8_dec_hw_ctx
182 * @segment_map: Segment map buffer.
183 * @prob_tbl: Probability table buffer.
185 struct hantro_vp8_dec_hw_ctx
{
186 struct hantro_aux_buf segment_map
;
187 struct hantro_aux_buf prob_tbl
;
191 * struct hantro_vp9_frame_info
193 * @valid: frame info valid flag
194 * @frame_context_idx: index of frame context
195 * @reference_mode: inter prediction type
196 * @tx_mode: transform mode
197 * @interpolation_filter: filter selection for inter prediction
198 * @flags: frame flags
199 * @timestamp: frame timestamp
201 struct hantro_vp9_frame_info
{
203 u32 frame_context_idx
: 2;
204 u32 reference_mode
: 2;
206 u32 interpolation_filter
: 3;
211 #define MAX_SB_COLS 64
212 #define MAX_SB_ROWS 34
215 * struct hantro_vp9_dec_hw_ctx
217 * @tile_edge: auxiliary DMA buffer for tile edge processing
218 * @segment_map: auxiliary DMA buffer for segment map
219 * @misc: auxiliary DMA buffer for tile info, probabilities and hw counters
220 * @cnts: vp9 library struct for abstracting hw counters access
221 * @probability_tables: VP9 probability tables implied by the spec
222 * @frame_context: VP9 frame contexts
223 * @cur: current frame information
224 * @last: last frame information
225 * @bsd_ctrl_offset: bsd offset into tile_edge
226 * @segment_map_size: size of segment map
227 * @ctx_counters_offset: hw counters offset into misc
228 * @tile_info_offset: tile info offset into misc
229 * @tile_r_info: per-tile information array
230 * @tile_c_info: per-tile information array
231 * @last_tile_r: last number of tile rows
232 * @last_tile_c: last number of tile cols
233 * @last_sbs_r: last number of superblock rows
234 * @last_sbs_c: last number of superblock cols
235 * @active_segment: number of active segment (alternating between 0 and 1)
236 * @feature_enabled: segmentation feature enabled flags
237 * @feature_data: segmentation feature data
239 struct hantro_vp9_dec_hw_ctx
{
240 struct hantro_aux_buf tile_edge
;
241 struct hantro_aux_buf segment_map
;
242 struct hantro_aux_buf misc
;
243 struct v4l2_vp9_frame_symbol_counts cnts
;
244 struct v4l2_vp9_frame_context probability_tables
;
245 struct v4l2_vp9_frame_context frame_context
[4];
246 struct hantro_vp9_frame_info cur
;
247 struct hantro_vp9_frame_info last
;
249 unsigned int bsd_ctrl_offset
;
250 unsigned int segment_map_size
;
251 unsigned int ctx_counters_offset
;
252 unsigned int tile_info_offset
;
254 unsigned short tile_r_info
[MAX_SB_ROWS
];
255 unsigned short tile_c_info
[MAX_SB_COLS
];
256 unsigned int last_tile_r
;
257 unsigned int last_tile_c
;
258 unsigned int last_sbs_r
;
259 unsigned int last_sbs_c
;
261 unsigned int active_segment
;
262 u8 feature_enabled
[8];
263 s16 feature_data
[8][4];
267 * struct hantro_av1_dec_ctrls
268 * @sequence: AV1 Sequence
269 * @tile_group_entry: AV1 Tile Group entry
270 * @frame: AV1 Frame Header OBU
271 * @film_grain: AV1 Film Grain
273 struct hantro_av1_dec_ctrls
{
274 const struct v4l2_ctrl_av1_sequence
*sequence
;
275 const struct v4l2_ctrl_av1_tile_group_entry
*tile_group_entry
;
276 const struct v4l2_ctrl_av1_frame
*frame
;
277 const struct v4l2_ctrl_av1_film_grain
*film_grain
;
280 struct hantro_av1_frame_ref
{
286 enum v4l2_av1_frame_type frame_type
;
289 u32 order_hints
[V4L2_AV1_TOTAL_REFS_PER_FRAME
];
290 struct vb2_v4l2_buffer
*vb2_ref
;
294 * struct hantro_av1_dec_hw_ctx
295 * @db_data_col: db tile col data buffer
296 * @db_ctrl_col: db tile col ctrl buffer
297 * @cdef_col: cdef tile col buffer
298 * @sr_col: sr tile col buffer
299 * @lr_col: lr tile col buffer
300 * @global_model: global model buffer
301 * @tile_info: tile info buffer
302 * @segment: segmentation info buffer
303 * @film_grain: film grain buffer
304 * @prob_tbl: probability table
305 * @prob_tbl_out: probability table output
306 * @tile_buf: tile buffer
307 * @ctrls: V4L2 controls attached to a run
308 * @frame_refs: reference frames info slots
309 * @ref_frame_sign_bias: array of sign bias
310 * @num_tile_cols_allocated: number of allocated tiles
311 * @cdfs: current probabilities structure
312 * @cdfs_ndvc: current mv probabilities structure
313 * @default_cdfs: default probabilities structure
314 * @default_cdfs_ndvc: default mv probabilties structure
315 * @cdfs_last: stored probabilities structures
316 * @cdfs_last_ndvc: stored mv probabilities structures
317 * @current_frame_index: index of the current in frame_refs array
319 struct hantro_av1_dec_hw_ctx
{
320 struct hantro_aux_buf db_data_col
;
321 struct hantro_aux_buf db_ctrl_col
;
322 struct hantro_aux_buf cdef_col
;
323 struct hantro_aux_buf sr_col
;
324 struct hantro_aux_buf lr_col
;
325 struct hantro_aux_buf global_model
;
326 struct hantro_aux_buf tile_info
;
327 struct hantro_aux_buf segment
;
328 struct hantro_aux_buf film_grain
;
329 struct hantro_aux_buf prob_tbl
;
330 struct hantro_aux_buf prob_tbl_out
;
331 struct hantro_aux_buf tile_buf
;
332 struct hantro_av1_dec_ctrls ctrls
;
333 struct hantro_av1_frame_ref frame_refs
[AV1_MAX_FRAME_BUF_COUNT
];
334 u32 ref_frame_sign_bias
[V4L2_AV1_TOTAL_REFS_PER_FRAME
];
335 unsigned int num_tile_cols_allocated
;
336 struct av1cdfs
*cdfs
;
337 struct mvcdfs
*cdfs_ndvc
;
338 struct av1cdfs default_cdfs
;
339 struct mvcdfs default_cdfs_ndvc
;
340 struct av1cdfs cdfs_last
[NUM_REF_FRAMES
];
341 struct mvcdfs cdfs_last_ndvc
[NUM_REF_FRAMES
];
342 int current_frame_index
;
345 * struct hantro_postproc_ctx
347 * @dec_q: References buffers, in decoder format.
349 struct hantro_postproc_ctx
{
350 struct hantro_aux_buf dec_q
[MAX_POSTPROC_BUFFERS
];
354 * struct hantro_postproc_ops - post-processor operations
356 * @enable: Enable the post-processor block. Optional.
357 * @disable: Disable the post-processor block. Optional.
358 * @enum_framesizes: Enumerate possible scaled output formats.
359 * Returns zero if OK, a negative value in error cases.
362 struct hantro_postproc_ops
{
363 void (*enable
)(struct hantro_ctx
*ctx
);
364 void (*disable
)(struct hantro_ctx
*ctx
);
365 int (*enum_framesizes
)(struct hantro_ctx
*ctx
, struct v4l2_frmsizeenum
*fsize
);
369 * struct hantro_codec_ops - codec mode specific operations
371 * @init: If needed, can be used for initialization.
372 * Optional and called from process context.
373 * @exit: If needed, can be used to undo the .init phase.
374 * Optional and called from process context.
375 * @run: Start single {en,de)coding job. Called from atomic context
376 * to indicate that a pair of buffers is ready and the hardware
377 * should be programmed and started. Returns zero if OK, a
378 * negative value in error cases.
379 * @done: Read back processing results and additional data from hardware.
380 * @reset: Reset the hardware in case of a timeout.
382 struct hantro_codec_ops
{
383 int (*init
)(struct hantro_ctx
*ctx
);
384 void (*exit
)(struct hantro_ctx
*ctx
);
385 int (*run
)(struct hantro_ctx
*ctx
);
386 void (*done
)(struct hantro_ctx
*ctx
);
387 void (*reset
)(struct hantro_ctx
*ctx
);
391 * enum hantro_enc_fmt - source format ID for hardware registers.
393 * @ROCKCHIP_VPU_ENC_FMT_YUV420P: Y/CbCr 4:2:0 planar format
394 * @ROCKCHIP_VPU_ENC_FMT_YUV420SP: Y/CbCr 4:2:0 semi-planar format
395 * @ROCKCHIP_VPU_ENC_FMT_YUYV422: YUV 4:2:2 packed format (YUYV)
396 * @ROCKCHIP_VPU_ENC_FMT_UYVY422: YUV 4:2:2 packed format (UYVY)
398 enum hantro_enc_fmt
{
399 ROCKCHIP_VPU_ENC_FMT_YUV420P
= 0,
400 ROCKCHIP_VPU_ENC_FMT_YUV420SP
= 1,
401 ROCKCHIP_VPU_ENC_FMT_YUYV422
= 2,
402 ROCKCHIP_VPU_ENC_FMT_UYVY422
= 3,
405 extern const struct hantro_variant imx8mm_vpu_g1_variant
;
406 extern const struct hantro_variant imx8mq_vpu_g1_variant
;
407 extern const struct hantro_variant imx8mq_vpu_g2_variant
;
408 extern const struct hantro_variant imx8mq_vpu_variant
;
409 extern const struct hantro_variant px30_vpu_variant
;
410 extern const struct hantro_variant rk3036_vpu_variant
;
411 extern const struct hantro_variant rk3066_vpu_variant
;
412 extern const struct hantro_variant rk3288_vpu_variant
;
413 extern const struct hantro_variant rk3328_vpu_variant
;
414 extern const struct hantro_variant rk3399_vpu_variant
;
415 extern const struct hantro_variant rk3568_vepu_variant
;
416 extern const struct hantro_variant rk3568_vpu_variant
;
417 extern const struct hantro_variant rk3588_vpu981_variant
;
418 extern const struct hantro_variant sama5d4_vdec_variant
;
419 extern const struct hantro_variant sunxi_vpu_variant
;
420 extern const struct hantro_variant stm32mp25_vdec_variant
;
421 extern const struct hantro_variant stm32mp25_venc_variant
;
423 extern const struct hantro_postproc_ops hantro_g1_postproc_ops
;
424 extern const struct hantro_postproc_ops hantro_g2_postproc_ops
;
425 extern const struct hantro_postproc_ops rockchip_vpu981_postproc_ops
;
427 extern const u32 hantro_vp8_dec_mc_filter
[8][6];
429 void hantro_watchdog(struct work_struct
*work
);
430 void hantro_run(struct hantro_ctx
*ctx
);
431 void hantro_irq_done(struct hantro_dev
*vpu
,
432 enum vb2_buffer_state result
);
433 void hantro_start_prepare_run(struct hantro_ctx
*ctx
);
434 void hantro_end_prepare_run(struct hantro_ctx
*ctx
);
436 irqreturn_t
hantro_g1_irq(int irq
, void *dev_id
);
437 void hantro_g1_reset(struct hantro_ctx
*ctx
);
439 int hantro_h1_jpeg_enc_run(struct hantro_ctx
*ctx
);
440 int rockchip_vpu2_jpeg_enc_run(struct hantro_ctx
*ctx
);
441 void hantro_h1_jpeg_enc_done(struct hantro_ctx
*ctx
);
442 void rockchip_vpu2_jpeg_enc_done(struct hantro_ctx
*ctx
);
444 dma_addr_t
hantro_h264_get_ref_buf(struct hantro_ctx
*ctx
,
445 unsigned int dpb_idx
);
446 u16
hantro_h264_get_ref_nbr(struct hantro_ctx
*ctx
,
447 unsigned int dpb_idx
);
448 int hantro_h264_dec_prepare_run(struct hantro_ctx
*ctx
);
449 int rockchip_vpu2_h264_dec_run(struct hantro_ctx
*ctx
);
450 int hantro_g1_h264_dec_run(struct hantro_ctx
*ctx
);
451 int hantro_h264_dec_init(struct hantro_ctx
*ctx
);
452 void hantro_h264_dec_exit(struct hantro_ctx
*ctx
);
454 int hantro_hevc_dec_init(struct hantro_ctx
*ctx
);
455 void hantro_hevc_dec_exit(struct hantro_ctx
*ctx
);
456 int hantro_g2_hevc_dec_run(struct hantro_ctx
*ctx
);
457 int hantro_hevc_dec_prepare_run(struct hantro_ctx
*ctx
);
458 void hantro_hevc_ref_init(struct hantro_ctx
*ctx
);
459 dma_addr_t
hantro_hevc_get_ref_buf(struct hantro_ctx
*ctx
, s32 poc
);
460 int hantro_hevc_add_ref_buf(struct hantro_ctx
*ctx
, int poc
, dma_addr_t addr
);
462 int rockchip_vpu981_av1_dec_init(struct hantro_ctx
*ctx
);
463 void rockchip_vpu981_av1_dec_exit(struct hantro_ctx
*ctx
);
464 int rockchip_vpu981_av1_dec_run(struct hantro_ctx
*ctx
);
465 void rockchip_vpu981_av1_dec_done(struct hantro_ctx
*ctx
);
467 static inline unsigned short hantro_vp9_num_sbs(unsigned short dimension
)
469 return (dimension
+ 63) / 64;
473 hantro_vp9_mv_size(unsigned int width
, unsigned int height
)
478 * There can be up to (CTBs x 64) number of blocks,
479 * and the motion vector for each block needs 16 bytes.
481 num_ctbs
= hantro_vp9_num_sbs(width
) * hantro_vp9_num_sbs(height
);
482 return (num_ctbs
* 64) * 16;
486 hantro_h264_mv_size(unsigned int width
, unsigned int height
)
489 * A decoded 8-bit 4:2:0 NV12 frame may need memory for up to
490 * 448 bytes per macroblock with additional 32 bytes on
491 * multi-core variants.
493 * The H264 decoder needs extra space on the output buffers
494 * to store motion vectors. This is needed for reference
495 * frames and only if the format is non-post-processed NV12.
497 * Memory layout is as follow:
499 * +---------------------------+
500 * | Y-plane 256 bytes x MBs |
501 * +---------------------------+
502 * | UV-plane 128 bytes x MBs |
503 * +---------------------------+
504 * | MV buffer 64 bytes x MBs |
505 * +---------------------------+
506 * | MC sync 32 bytes |
507 * +---------------------------+
509 return 64 * MB_WIDTH(width
) * MB_WIDTH(height
) + 32;
513 hantro_hevc_mv_size(unsigned int width
, unsigned int height
)
516 * A CTB can be 64x64, 32x32 or 16x16.
517 * Allocated memory for the "worse" case: 16x16
519 return width
* height
/ 16;
523 hantro_hevc_luma_compressed_size(unsigned int width
, unsigned int height
)
525 u32 pic_width_in_cbsy
=
526 round_up((width
+ CBS_LUMA
- 1) / CBS_LUMA
, CBS_SIZE
);
527 u32 pic_height_in_cbsy
= (height
+ CBS_LUMA
- 1) / CBS_LUMA
;
529 return round_up(pic_width_in_cbsy
* pic_height_in_cbsy
, CBS_SIZE
);
533 hantro_hevc_chroma_compressed_size(unsigned int width
, unsigned int height
)
535 u32 pic_width_in_cbsc
=
536 round_up((width
+ CBS_CHROMA_W
- 1) / CBS_CHROMA_W
, CBS_SIZE
);
537 u32 pic_height_in_cbsc
= (height
/ 2 + CBS_CHROMA_H
- 1) / CBS_CHROMA_H
;
539 return round_up(pic_width_in_cbsc
* pic_height_in_cbsc
, CBS_SIZE
);
543 hantro_hevc_compressed_size(unsigned int width
, unsigned int height
)
545 return hantro_hevc_luma_compressed_size(width
, height
) +
546 hantro_hevc_chroma_compressed_size(width
, height
);
549 static inline unsigned short hantro_av1_num_sbs(unsigned short dimension
)
551 return DIV_ROUND_UP(dimension
, 64);
555 hantro_av1_mv_size(unsigned int width
, unsigned int height
)
557 size_t num_sbs
= hantro_av1_num_sbs(width
) * hantro_av1_num_sbs(height
);
559 return ALIGN(num_sbs
* 384, 16) * 2 + 512;
562 size_t hantro_g2_chroma_offset(struct hantro_ctx
*ctx
);
563 size_t hantro_g2_motion_vectors_offset(struct hantro_ctx
*ctx
);
564 size_t hantro_g2_luma_compress_offset(struct hantro_ctx
*ctx
);
565 size_t hantro_g2_chroma_compress_offset(struct hantro_ctx
*ctx
);
567 int hantro_g1_mpeg2_dec_run(struct hantro_ctx
*ctx
);
568 int rockchip_vpu2_mpeg2_dec_run(struct hantro_ctx
*ctx
);
569 void hantro_mpeg2_dec_copy_qtable(u8
*qtable
,
570 const struct v4l2_ctrl_mpeg2_quantisation
*ctrl
);
571 int hantro_mpeg2_dec_init(struct hantro_ctx
*ctx
);
572 void hantro_mpeg2_dec_exit(struct hantro_ctx
*ctx
);
574 int hantro_g1_vp8_dec_run(struct hantro_ctx
*ctx
);
575 int rockchip_vpu2_vp8_dec_run(struct hantro_ctx
*ctx
);
576 int hantro_vp8_dec_init(struct hantro_ctx
*ctx
);
577 void hantro_vp8_dec_exit(struct hantro_ctx
*ctx
);
578 void hantro_vp8_prob_update(struct hantro_ctx
*ctx
,
579 const struct v4l2_ctrl_vp8_frame
*hdr
);
581 int hantro_g2_vp9_dec_run(struct hantro_ctx
*ctx
);
582 void hantro_g2_vp9_dec_done(struct hantro_ctx
*ctx
);
583 int hantro_vp9_dec_init(struct hantro_ctx
*ctx
);
584 void hantro_vp9_dec_exit(struct hantro_ctx
*ctx
);
585 void hantro_g2_check_idle(struct hantro_dev
*vpu
);
586 irqreturn_t
hantro_g2_irq(int irq
, void *dev_id
);
588 #endif /* HANTRO_HW_H_ */