2 * RockChip MPP Video Decoder
3 * Copyright (c) 2017 Lionel CHAZALLON
5 * This file is part of FFmpeg.
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 #include <drm_fourcc.h>
24 #include <rockchip/mpp_buffer.h>
25 #include <rockchip/rk_mpi.h>
30 #include "codec_internal.h"
33 #include "libavutil/refstruct.h"
34 #include "libavutil/buffer.h"
35 #include "libavutil/common.h"
36 #include "libavutil/frame.h"
37 #include "libavutil/hwcontext.h"
38 #include "libavutil/hwcontext_drm.h"
39 #include "libavutil/imgutils.h"
40 #include "libavutil/log.h"
41 #include "libavutil/mem.h"
43 #define RECEIVE_FRAME_TIMEOUT 100
44 #define FRAMEGROUP_MAX_FRAMES 16
45 #define INPUT_MAX_PACKETS 4
50 MppBufferGroup frame_group
;
55 AVBufferRef
*frames_ref
;
56 AVBufferRef
*device_ref
;
61 RKMPPDecoder
*decoder
; ///< RefStruct reference
66 const RKMPPDecoder
*decoder_ref
; ///< RefStruct reference
69 static MppCodingType
rkmpp_get_codingtype(AVCodecContext
*avctx
)
71 switch (avctx
->codec_id
) {
72 case AV_CODEC_ID_H264
: return MPP_VIDEO_CodingAVC
;
73 case AV_CODEC_ID_HEVC
: return MPP_VIDEO_CodingHEVC
;
74 case AV_CODEC_ID_VP8
: return MPP_VIDEO_CodingVP8
;
75 case AV_CODEC_ID_VP9
: return MPP_VIDEO_CodingVP9
;
76 default: return MPP_VIDEO_CodingUnused
;
80 static uint32_t rkmpp_get_frameformat(MppFrameFormat mppformat
)
83 case MPP_FMT_YUV420SP
: return DRM_FORMAT_NV12
;
84 #ifdef DRM_FORMAT_NV12_10
85 case MPP_FMT_YUV420SP_10BIT
: return DRM_FORMAT_NV12_10
;
91 static int rkmpp_write_data(AVCodecContext
*avctx
, uint8_t *buffer
, int size
, int64_t pts
)
93 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
94 RKMPPDecoder
*decoder
= rk_context
->decoder
;
98 // create the MPP packet
99 ret
= mpp_packet_init(&packet
, buffer
, size
);
101 av_log(avctx
, AV_LOG_ERROR
, "Failed to init MPP packet (code = %d)\n", ret
);
102 return AVERROR_UNKNOWN
;
105 mpp_packet_set_pts(packet
, pts
);
108 mpp_packet_set_eos(packet
);
110 ret
= decoder
->mpi
->decode_put_packet(decoder
->ctx
, packet
);
112 if (ret
== MPP_ERR_BUFFER_FULL
) {
113 av_log(avctx
, AV_LOG_DEBUG
, "Buffer full writing %d bytes to decoder\n", size
);
114 ret
= AVERROR(EAGAIN
);
116 ret
= AVERROR_UNKNOWN
;
119 av_log(avctx
, AV_LOG_DEBUG
, "Wrote %d bytes to decoder\n", size
);
121 mpp_packet_deinit(&packet
);
126 static int rkmpp_close_decoder(AVCodecContext
*avctx
)
128 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
129 av_refstruct_unref(&rk_context
->decoder
);
133 static void rkmpp_release_decoder(AVRefStructOpaque unused
, void *obj
)
135 RKMPPDecoder
*decoder
= obj
;
138 decoder
->mpi
->reset(decoder
->ctx
);
139 mpp_destroy(decoder
->ctx
);
143 if (decoder
->frame_group
) {
144 mpp_buffer_group_put(decoder
->frame_group
);
145 decoder
->frame_group
= NULL
;
148 av_buffer_unref(&decoder
->frames_ref
);
149 av_buffer_unref(&decoder
->device_ref
);
152 static int rkmpp_init_decoder(AVCodecContext
*avctx
)
154 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
155 RKMPPDecoder
*decoder
= NULL
;
156 MppCodingType codectype
= MPP_VIDEO_CodingUnused
;
161 avctx
->pix_fmt
= AV_PIX_FMT_DRM_PRIME
;
163 // create a decoder and a ref to it
164 decoder
= av_refstruct_alloc_ext(sizeof(*decoder
), 0,
165 NULL
, rkmpp_release_decoder
);
167 ret
= AVERROR(ENOMEM
);
170 rk_context
->decoder
= decoder
;
172 av_log(avctx
, AV_LOG_DEBUG
, "Initializing RKMPP decoder.\n");
174 codectype
= rkmpp_get_codingtype(avctx
);
175 if (codectype
== MPP_VIDEO_CodingUnused
) {
176 av_log(avctx
, AV_LOG_ERROR
, "Unknown codec type (%d).\n", avctx
->codec_id
);
177 ret
= AVERROR_UNKNOWN
;
181 ret
= mpp_check_support_format(MPP_CTX_DEC
, codectype
);
183 av_log(avctx
, AV_LOG_ERROR
, "Codec type (%d) unsupported by MPP\n", avctx
->codec_id
);
184 ret
= AVERROR_UNKNOWN
;
188 // Create the MPP context
189 ret
= mpp_create(&decoder
->ctx
, &decoder
->mpi
);
191 av_log(avctx
, AV_LOG_ERROR
, "Failed to create MPP context (code = %d).\n", ret
);
192 ret
= AVERROR_UNKNOWN
;
197 ret
= mpp_init(decoder
->ctx
, MPP_CTX_DEC
, codectype
);
199 av_log(avctx
, AV_LOG_ERROR
, "Failed to initialize MPP context (code = %d).\n", ret
);
200 ret
= AVERROR_UNKNOWN
;
204 // make decode calls blocking with a timeout
205 paramS32
= MPP_POLL_BLOCK
;
206 ret
= decoder
->mpi
->control(decoder
->ctx
, MPP_SET_OUTPUT_BLOCK
, ¶mS32
);
208 av_log(avctx
, AV_LOG_ERROR
, "Failed to set blocking mode on MPI (code = %d).\n", ret
);
209 ret
= AVERROR_UNKNOWN
;
213 paramS64
= RECEIVE_FRAME_TIMEOUT
;
214 ret
= decoder
->mpi
->control(decoder
->ctx
, MPP_SET_OUTPUT_BLOCK_TIMEOUT
, ¶mS64
);
216 av_log(avctx
, AV_LOG_ERROR
, "Failed to set block timeout on MPI (code = %d).\n", ret
);
217 ret
= AVERROR_UNKNOWN
;
221 ret
= mpp_buffer_group_get_internal(&decoder
->frame_group
, MPP_BUFFER_TYPE_ION
);
223 av_log(avctx
, AV_LOG_ERROR
, "Failed to retrieve buffer group (code = %d)\n", ret
);
224 ret
= AVERROR_UNKNOWN
;
228 ret
= decoder
->mpi
->control(decoder
->ctx
, MPP_DEC_SET_EXT_BUF_GROUP
, decoder
->frame_group
);
230 av_log(avctx
, AV_LOG_ERROR
, "Failed to assign buffer group (code = %d)\n", ret
);
231 ret
= AVERROR_UNKNOWN
;
235 ret
= mpp_buffer_group_limit_config(decoder
->frame_group
, 0, FRAMEGROUP_MAX_FRAMES
);
237 av_log(avctx
, AV_LOG_ERROR
, "Failed to set buffer group limit (code = %d)\n", ret
);
238 ret
= AVERROR_UNKNOWN
;
242 decoder
->first_packet
= 1;
244 av_log(avctx
, AV_LOG_DEBUG
, "RKMPP decoder initialized successfully.\n");
246 decoder
->device_ref
= av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM
);
247 if (!decoder
->device_ref
) {
248 ret
= AVERROR(ENOMEM
);
251 ret
= av_hwdevice_ctx_init(decoder
->device_ref
);
258 av_log(avctx
, AV_LOG_ERROR
, "Failed to initialize RKMPP decoder.\n");
259 rkmpp_close_decoder(avctx
);
263 static int rkmpp_send_packet(AVCodecContext
*avctx
, const AVPacket
*avpkt
)
265 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
266 RKMPPDecoder
*decoder
= rk_context
->decoder
;
271 av_log(avctx
, AV_LOG_DEBUG
, "End of stream.\n");
272 decoder
->eos_reached
= 1;
273 ret
= rkmpp_write_data(avctx
, NULL
, 0, 0);
275 av_log(avctx
, AV_LOG_ERROR
, "Failed to send EOS to decoder (code = %d)\n", ret
);
279 // on first packet, send extradata
280 if (decoder
->first_packet
) {
281 if (avctx
->extradata_size
) {
282 ret
= rkmpp_write_data(avctx
, avctx
->extradata
,
283 avctx
->extradata_size
,
286 av_log(avctx
, AV_LOG_ERROR
, "Failed to write extradata to decoder (code = %d)\n", ret
);
290 decoder
->first_packet
= 0;
294 ret
= rkmpp_write_data(avctx
, avpkt
->data
, avpkt
->size
, avpkt
->pts
);
295 if (ret
&& ret
!=AVERROR(EAGAIN
))
296 av_log(avctx
, AV_LOG_ERROR
, "Failed to write data to decoder (code = %d)\n", ret
);
301 static void rkmpp_release_frame(void *opaque
, uint8_t *data
)
303 AVDRMFrameDescriptor
*desc
= (AVDRMFrameDescriptor
*)data
;
304 RKMPPFrameContext
*framecontext
= opaque
;
306 mpp_frame_deinit(&framecontext
->frame
);
307 av_refstruct_unref(&framecontext
->decoder_ref
);
312 static int rkmpp_retrieve_frame(AVCodecContext
*avctx
, AVFrame
*frame
)
314 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
315 RKMPPDecoder
*decoder
= rk_context
->decoder
;
317 MppFrame mppframe
= NULL
;
318 MppBuffer buffer
= NULL
;
319 AVDRMLayerDescriptor
*layer
= NULL
;
321 MppFrameFormat mppformat
;
324 ret
= decoder
->mpi
->decode_get_frame(decoder
->ctx
, &mppframe
);
325 if (ret
!= MPP_OK
&& ret
!= MPP_ERR_TIMEOUT
) {
326 av_log(avctx
, AV_LOG_ERROR
, "Failed to get a frame from MPP (code = %d)\n", ret
);
331 // Check whether we have a special frame or not
332 if (mpp_frame_get_info_change(mppframe
)) {
333 AVHWFramesContext
*hwframes
;
335 av_log(avctx
, AV_LOG_INFO
, "Decoder noticed an info change (%dx%d), format=%d\n",
336 (int)mpp_frame_get_width(mppframe
), (int)mpp_frame_get_height(mppframe
),
337 (int)mpp_frame_get_fmt(mppframe
));
339 avctx
->width
= mpp_frame_get_width(mppframe
);
340 avctx
->height
= mpp_frame_get_height(mppframe
);
342 decoder
->mpi
->control(decoder
->ctx
, MPP_DEC_SET_INFO_CHANGE_READY
, NULL
);
344 av_buffer_unref(&decoder
->frames_ref
);
346 decoder
->frames_ref
= av_hwframe_ctx_alloc(decoder
->device_ref
);
347 if (!decoder
->frames_ref
) {
348 ret
= AVERROR(ENOMEM
);
352 mppformat
= mpp_frame_get_fmt(mppframe
);
353 drmformat
= rkmpp_get_frameformat(mppformat
);
355 hwframes
= (AVHWFramesContext
*)decoder
->frames_ref
->data
;
356 hwframes
->format
= AV_PIX_FMT_DRM_PRIME
;
357 hwframes
->sw_format
= drmformat
== DRM_FORMAT_NV12
? AV_PIX_FMT_NV12
: AV_PIX_FMT_NONE
;
358 hwframes
->width
= avctx
->width
;
359 hwframes
->height
= avctx
->height
;
360 ret
= av_hwframe_ctx_init(decoder
->frames_ref
);
364 // here decoder is fully initialized, we need to feed it again with data
365 ret
= AVERROR(EAGAIN
);
367 } else if (mpp_frame_get_eos(mppframe
)) {
368 av_log(avctx
, AV_LOG_DEBUG
, "Received a EOS frame.\n");
369 decoder
->eos_reached
= 1;
372 } else if (mpp_frame_get_discard(mppframe
)) {
373 av_log(avctx
, AV_LOG_DEBUG
, "Received a discard frame.\n");
374 ret
= AVERROR(EAGAIN
);
376 } else if (mpp_frame_get_errinfo(mppframe
)) {
377 av_log(avctx
, AV_LOG_ERROR
, "Received a errinfo frame.\n");
378 ret
= AVERROR_UNKNOWN
;
382 // here we should have a valid frame
383 av_log(avctx
, AV_LOG_DEBUG
, "Received a frame.\n");
385 // setup general frame fields
386 frame
->format
= AV_PIX_FMT_DRM_PRIME
;
387 frame
->width
= mpp_frame_get_width(mppframe
);
388 frame
->height
= mpp_frame_get_height(mppframe
);
389 frame
->pts
= mpp_frame_get_pts(mppframe
);
390 frame
->color_range
= mpp_frame_get_color_range(mppframe
);
391 frame
->color_primaries
= mpp_frame_get_color_primaries(mppframe
);
392 frame
->color_trc
= mpp_frame_get_color_trc(mppframe
);
393 frame
->colorspace
= mpp_frame_get_colorspace(mppframe
);
395 mode
= mpp_frame_get_mode(mppframe
);
396 if ((mode
& MPP_FRAME_FLAG_FIELD_ORDER_MASK
) == MPP_FRAME_FLAG_DEINTERLACED
)
397 frame
->flags
|= AV_FRAME_FLAG_INTERLACED
;
398 if ((mode
& MPP_FRAME_FLAG_FIELD_ORDER_MASK
) == MPP_FRAME_FLAG_TOP_FIRST
)
399 frame
->flags
|= AV_FRAME_FLAG_TOP_FIELD_FIRST
;
401 mppformat
= mpp_frame_get_fmt(mppframe
);
402 drmformat
= rkmpp_get_frameformat(mppformat
);
404 // now setup the frame buffer info
405 buffer
= mpp_frame_get_buffer(mppframe
);
407 RKMPPFrameContext
*framecontext
;
408 AVDRMFrameDescriptor
*desc
;
409 // We allocate the descriptor in buf[0] jointly with a structure
410 // that will allow to hold additional information
411 // for properly releasing MPP frames and decoder.
413 AVDRMFrameDescriptor desc
;
414 RKMPPFrameContext framecontext
;
415 } *combined_desc
= av_mallocz(sizeof(*combined_desc
));
416 if (!combined_desc
) {
417 ret
= AVERROR(ENOMEM
);
420 desc
= &combined_desc
->desc
;
421 framecontext
= &combined_desc
->framecontext
;
423 desc
->nb_objects
= 1;
424 desc
->objects
[0].fd
= mpp_buffer_get_fd(buffer
);
425 desc
->objects
[0].size
= mpp_buffer_get_size(buffer
);
428 layer
= &desc
->layers
[0];
429 layer
->format
= drmformat
;
430 layer
->nb_planes
= 2;
432 layer
->planes
[0].object_index
= 0;
433 layer
->planes
[0].offset
= 0;
434 layer
->planes
[0].pitch
= mpp_frame_get_hor_stride(mppframe
);
436 layer
->planes
[1].object_index
= 0;
437 layer
->planes
[1].offset
= layer
->planes
[0].pitch
* mpp_frame_get_ver_stride(mppframe
);
438 layer
->planes
[1].pitch
= layer
->planes
[0].pitch
;
440 // MPP decoder needs to be closed only when all frames have been released.
441 framecontext
->frame
= mppframe
;
443 frame
->data
[0] = (uint8_t *)desc
;
444 frame
->buf
[0] = av_buffer_create((uint8_t *)desc
, sizeof(*desc
), rkmpp_release_frame
,
445 framecontext
, AV_BUFFER_FLAG_READONLY
);
447 if (!frame
->buf
[0]) {
448 av_free(combined_desc
);
449 ret
= AVERROR(ENOMEM
);
452 framecontext
->decoder_ref
= av_refstruct_ref(rk_context
->decoder
);
454 frame
->hw_frames_ctx
= av_buffer_ref(decoder
->frames_ref
);
455 if (!frame
->hw_frames_ctx
) {
456 av_frame_unref(frame
);
457 return AVERROR(ENOMEM
);
462 av_log(avctx
, AV_LOG_ERROR
, "Failed to retrieve the frame buffer, frame is dropped (code = %d)\n", ret
);
463 mpp_frame_deinit(&mppframe
);
465 } else if (decoder
->eos_reached
) {
467 } else if (ret
== MPP_ERR_TIMEOUT
) {
468 av_log(avctx
, AV_LOG_DEBUG
, "Timeout when trying to get a frame from MPP\n");
471 return AVERROR(EAGAIN
);
475 mpp_frame_deinit(&mppframe
);
480 static int rkmpp_receive_frame(AVCodecContext
*avctx
, AVFrame
*frame
)
482 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
483 RKMPPDecoder
*decoder
= rk_context
->decoder
;
486 RK_S32 usedslots
, freeslots
;
488 if (!decoder
->eos_reached
) {
489 // we get the available slots in decoder
490 ret
= decoder
->mpi
->control(decoder
->ctx
, MPP_DEC_GET_STREAM_COUNT
, &usedslots
);
492 av_log(avctx
, AV_LOG_ERROR
, "Failed to get decoder used slots (code = %d).\n", ret
);
496 freeslots
= INPUT_MAX_PACKETS
- usedslots
;
498 ret
= ff_decode_get_packet(avctx
, &pkt
);
499 if (ret
< 0 && ret
!= AVERROR_EOF
) {
503 ret
= rkmpp_send_packet(avctx
, &pkt
);
504 av_packet_unref(&pkt
);
507 av_log(avctx
, AV_LOG_ERROR
, "Failed to send packet to decoder (code = %d)\n", ret
);
512 // make sure we keep decoder full
514 return AVERROR(EAGAIN
);
517 return rkmpp_retrieve_frame(avctx
, frame
);
520 static void rkmpp_flush(AVCodecContext
*avctx
)
522 RKMPPDecodeContext
*rk_context
= avctx
->priv_data
;
523 RKMPPDecoder
*decoder
= rk_context
->decoder
;
526 av_log(avctx
, AV_LOG_DEBUG
, "Flush.\n");
528 ret
= decoder
->mpi
->reset(decoder
->ctx
);
530 decoder
->first_packet
= 1;
532 av_log(avctx
, AV_LOG_ERROR
, "Failed to reset MPI (code = %d)\n", ret
);
535 static const AVCodecHWConfigInternal
*const rkmpp_hw_configs
[] = {
536 HW_CONFIG_INTERNAL(DRM_PRIME
),
540 #define RKMPP_DEC_CLASS(NAME) \
541 static const AVClass rkmpp_##NAME##_dec_class = { \
542 .class_name = "rkmpp_" #NAME "_dec", \
543 .version = LIBAVUTIL_VERSION_INT, \
546 #define RKMPP_DEC(NAME, ID, BSFS) \
547 RKMPP_DEC_CLASS(NAME) \
548 const FFCodec ff_##NAME##_rkmpp_decoder = { \
549 .p.name = #NAME "_rkmpp", \
550 CODEC_LONG_NAME(#NAME " (rkmpp)"), \
551 .p.type = AVMEDIA_TYPE_VIDEO, \
553 .priv_data_size = sizeof(RKMPPDecodeContext), \
554 .init = rkmpp_init_decoder, \
555 .close = rkmpp_close_decoder, \
556 FF_CODEC_RECEIVE_FRAME_CB(rkmpp_receive_frame), \
557 .flush = rkmpp_flush, \
558 .p.priv_class = &rkmpp_##NAME##_dec_class, \
559 .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_HARDWARE, \
560 .hw_configs = rkmpp_hw_configs, \
562 .p.wrapper_name = "rkmpp", \
563 .caps_internal = FF_CODEC_CAP_NOT_INIT_THREADSAFE, \
566 RKMPP_DEC(h264
, AV_CODEC_ID_H264
, "h264_mp4toannexb")
567 RKMPP_DEC(hevc
, AV_CODEC_ID_HEVC
, "hevc_mp4toannexb")
568 RKMPP_DEC(vp8
, AV_CODEC_ID_VP8
, NULL
)
569 RKMPP_DEC(vp9
, AV_CODEC_ID_VP9
, NULL
)