avformat/mpeg: demux ivtv captions
[ffmpeg.git] / libavcodec / rkmppdec.c
blob07d299f14439d49132e1bb4d6fe4060d351c9c7d
1 /*
2 * RockChip MPP Video Decoder
3 * Copyright (c) 2017 Lionel CHAZALLON
5 * This file is part of FFmpeg.
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 #include <drm_fourcc.h>
23 #include <pthread.h>
24 #include <rockchip/mpp_buffer.h>
25 #include <rockchip/rk_mpi.h>
26 #include <time.h>
27 #include <unistd.h>
29 #include "avcodec.h"
30 #include "codec_internal.h"
31 #include "decode.h"
32 #include "hwconfig.h"
33 #include "libavutil/refstruct.h"
34 #include "libavutil/buffer.h"
35 #include "libavutil/common.h"
36 #include "libavutil/frame.h"
37 #include "libavutil/hwcontext.h"
38 #include "libavutil/hwcontext_drm.h"
39 #include "libavutil/imgutils.h"
40 #include "libavutil/log.h"
41 #include "libavutil/mem.h"
43 #define RECEIVE_FRAME_TIMEOUT 100
44 #define FRAMEGROUP_MAX_FRAMES 16
45 #define INPUT_MAX_PACKETS 4
47 typedef struct {
48 MppCtx ctx;
49 MppApi *mpi;
50 MppBufferGroup frame_group;
52 char first_packet;
53 char eos_reached;
55 AVBufferRef *frames_ref;
56 AVBufferRef *device_ref;
57 } RKMPPDecoder;
59 typedef struct {
60 AVClass *av_class;
61 RKMPPDecoder *decoder; ///< RefStruct reference
62 } RKMPPDecodeContext;
64 typedef struct {
65 MppFrame frame;
66 const RKMPPDecoder *decoder_ref; ///< RefStruct reference
67 } RKMPPFrameContext;
69 static MppCodingType rkmpp_get_codingtype(AVCodecContext *avctx)
71 switch (avctx->codec_id) {
72 case AV_CODEC_ID_H264: return MPP_VIDEO_CodingAVC;
73 case AV_CODEC_ID_HEVC: return MPP_VIDEO_CodingHEVC;
74 case AV_CODEC_ID_VP8: return MPP_VIDEO_CodingVP8;
75 case AV_CODEC_ID_VP9: return MPP_VIDEO_CodingVP9;
76 default: return MPP_VIDEO_CodingUnused;
80 static uint32_t rkmpp_get_frameformat(MppFrameFormat mppformat)
82 switch (mppformat) {
83 case MPP_FMT_YUV420SP: return DRM_FORMAT_NV12;
84 #ifdef DRM_FORMAT_NV12_10
85 case MPP_FMT_YUV420SP_10BIT: return DRM_FORMAT_NV12_10;
86 #endif
87 default: return 0;
91 static int rkmpp_write_data(AVCodecContext *avctx, uint8_t *buffer, int size, int64_t pts)
93 RKMPPDecodeContext *rk_context = avctx->priv_data;
94 RKMPPDecoder *decoder = rk_context->decoder;
95 int ret;
96 MppPacket packet;
98 // create the MPP packet
99 ret = mpp_packet_init(&packet, buffer, size);
100 if (ret != MPP_OK) {
101 av_log(avctx, AV_LOG_ERROR, "Failed to init MPP packet (code = %d)\n", ret);
102 return AVERROR_UNKNOWN;
105 mpp_packet_set_pts(packet, pts);
107 if (!buffer)
108 mpp_packet_set_eos(packet);
110 ret = decoder->mpi->decode_put_packet(decoder->ctx, packet);
111 if (ret != MPP_OK) {
112 if (ret == MPP_ERR_BUFFER_FULL) {
113 av_log(avctx, AV_LOG_DEBUG, "Buffer full writing %d bytes to decoder\n", size);
114 ret = AVERROR(EAGAIN);
115 } else
116 ret = AVERROR_UNKNOWN;
118 else
119 av_log(avctx, AV_LOG_DEBUG, "Wrote %d bytes to decoder\n", size);
121 mpp_packet_deinit(&packet);
123 return ret;
126 static int rkmpp_close_decoder(AVCodecContext *avctx)
128 RKMPPDecodeContext *rk_context = avctx->priv_data;
129 av_refstruct_unref(&rk_context->decoder);
130 return 0;
133 static void rkmpp_release_decoder(AVRefStructOpaque unused, void *obj)
135 RKMPPDecoder *decoder = obj;
137 if (decoder->mpi) {
138 decoder->mpi->reset(decoder->ctx);
139 mpp_destroy(decoder->ctx);
140 decoder->ctx = NULL;
143 if (decoder->frame_group) {
144 mpp_buffer_group_put(decoder->frame_group);
145 decoder->frame_group = NULL;
148 av_buffer_unref(&decoder->frames_ref);
149 av_buffer_unref(&decoder->device_ref);
152 static int rkmpp_init_decoder(AVCodecContext *avctx)
154 RKMPPDecodeContext *rk_context = avctx->priv_data;
155 RKMPPDecoder *decoder = NULL;
156 MppCodingType codectype = MPP_VIDEO_CodingUnused;
157 int ret;
158 RK_S64 paramS64;
159 RK_S32 paramS32;
161 avctx->pix_fmt = AV_PIX_FMT_DRM_PRIME;
163 // create a decoder and a ref to it
164 decoder = av_refstruct_alloc_ext(sizeof(*decoder), 0,
165 NULL, rkmpp_release_decoder);
166 if (!decoder) {
167 ret = AVERROR(ENOMEM);
168 goto fail;
170 rk_context->decoder = decoder;
172 av_log(avctx, AV_LOG_DEBUG, "Initializing RKMPP decoder.\n");
174 codectype = rkmpp_get_codingtype(avctx);
175 if (codectype == MPP_VIDEO_CodingUnused) {
176 av_log(avctx, AV_LOG_ERROR, "Unknown codec type (%d).\n", avctx->codec_id);
177 ret = AVERROR_UNKNOWN;
178 goto fail;
181 ret = mpp_check_support_format(MPP_CTX_DEC, codectype);
182 if (ret != MPP_OK) {
183 av_log(avctx, AV_LOG_ERROR, "Codec type (%d) unsupported by MPP\n", avctx->codec_id);
184 ret = AVERROR_UNKNOWN;
185 goto fail;
188 // Create the MPP context
189 ret = mpp_create(&decoder->ctx, &decoder->mpi);
190 if (ret != MPP_OK) {
191 av_log(avctx, AV_LOG_ERROR, "Failed to create MPP context (code = %d).\n", ret);
192 ret = AVERROR_UNKNOWN;
193 goto fail;
196 // initialize mpp
197 ret = mpp_init(decoder->ctx, MPP_CTX_DEC, codectype);
198 if (ret != MPP_OK) {
199 av_log(avctx, AV_LOG_ERROR, "Failed to initialize MPP context (code = %d).\n", ret);
200 ret = AVERROR_UNKNOWN;
201 goto fail;
204 // make decode calls blocking with a timeout
205 paramS32 = MPP_POLL_BLOCK;
206 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK, &paramS32);
207 if (ret != MPP_OK) {
208 av_log(avctx, AV_LOG_ERROR, "Failed to set blocking mode on MPI (code = %d).\n", ret);
209 ret = AVERROR_UNKNOWN;
210 goto fail;
213 paramS64 = RECEIVE_FRAME_TIMEOUT;
214 ret = decoder->mpi->control(decoder->ctx, MPP_SET_OUTPUT_BLOCK_TIMEOUT, &paramS64);
215 if (ret != MPP_OK) {
216 av_log(avctx, AV_LOG_ERROR, "Failed to set block timeout on MPI (code = %d).\n", ret);
217 ret = AVERROR_UNKNOWN;
218 goto fail;
221 ret = mpp_buffer_group_get_internal(&decoder->frame_group, MPP_BUFFER_TYPE_ION);
222 if (ret) {
223 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve buffer group (code = %d)\n", ret);
224 ret = AVERROR_UNKNOWN;
225 goto fail;
228 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_SET_EXT_BUF_GROUP, decoder->frame_group);
229 if (ret) {
230 av_log(avctx, AV_LOG_ERROR, "Failed to assign buffer group (code = %d)\n", ret);
231 ret = AVERROR_UNKNOWN;
232 goto fail;
235 ret = mpp_buffer_group_limit_config(decoder->frame_group, 0, FRAMEGROUP_MAX_FRAMES);
236 if (ret) {
237 av_log(avctx, AV_LOG_ERROR, "Failed to set buffer group limit (code = %d)\n", ret);
238 ret = AVERROR_UNKNOWN;
239 goto fail;
242 decoder->first_packet = 1;
244 av_log(avctx, AV_LOG_DEBUG, "RKMPP decoder initialized successfully.\n");
246 decoder->device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_DRM);
247 if (!decoder->device_ref) {
248 ret = AVERROR(ENOMEM);
249 goto fail;
251 ret = av_hwdevice_ctx_init(decoder->device_ref);
252 if (ret < 0)
253 goto fail;
255 return 0;
257 fail:
258 av_log(avctx, AV_LOG_ERROR, "Failed to initialize RKMPP decoder.\n");
259 rkmpp_close_decoder(avctx);
260 return ret;
263 static int rkmpp_send_packet(AVCodecContext *avctx, const AVPacket *avpkt)
265 RKMPPDecodeContext *rk_context = avctx->priv_data;
266 RKMPPDecoder *decoder = rk_context->decoder;
267 int ret;
269 // handle EOF
270 if (!avpkt->size) {
271 av_log(avctx, AV_LOG_DEBUG, "End of stream.\n");
272 decoder->eos_reached = 1;
273 ret = rkmpp_write_data(avctx, NULL, 0, 0);
274 if (ret)
275 av_log(avctx, AV_LOG_ERROR, "Failed to send EOS to decoder (code = %d)\n", ret);
276 return ret;
279 // on first packet, send extradata
280 if (decoder->first_packet) {
281 if (avctx->extradata_size) {
282 ret = rkmpp_write_data(avctx, avctx->extradata,
283 avctx->extradata_size,
284 avpkt->pts);
285 if (ret) {
286 av_log(avctx, AV_LOG_ERROR, "Failed to write extradata to decoder (code = %d)\n", ret);
287 return ret;
290 decoder->first_packet = 0;
293 // now send packet
294 ret = rkmpp_write_data(avctx, avpkt->data, avpkt->size, avpkt->pts);
295 if (ret && ret!=AVERROR(EAGAIN))
296 av_log(avctx, AV_LOG_ERROR, "Failed to write data to decoder (code = %d)\n", ret);
298 return ret;
301 static void rkmpp_release_frame(void *opaque, uint8_t *data)
303 AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)data;
304 RKMPPFrameContext *framecontext = opaque;
306 mpp_frame_deinit(&framecontext->frame);
307 av_refstruct_unref(&framecontext->decoder_ref);
309 av_free(desc);
312 static int rkmpp_retrieve_frame(AVCodecContext *avctx, AVFrame *frame)
314 RKMPPDecodeContext *rk_context = avctx->priv_data;
315 RKMPPDecoder *decoder = rk_context->decoder;
316 int ret;
317 MppFrame mppframe = NULL;
318 MppBuffer buffer = NULL;
319 AVDRMLayerDescriptor *layer = NULL;
320 int mode;
321 MppFrameFormat mppformat;
322 uint32_t drmformat;
324 ret = decoder->mpi->decode_get_frame(decoder->ctx, &mppframe);
325 if (ret != MPP_OK && ret != MPP_ERR_TIMEOUT) {
326 av_log(avctx, AV_LOG_ERROR, "Failed to get a frame from MPP (code = %d)\n", ret);
327 goto fail;
330 if (mppframe) {
331 // Check whether we have a special frame or not
332 if (mpp_frame_get_info_change(mppframe)) {
333 AVHWFramesContext *hwframes;
335 av_log(avctx, AV_LOG_INFO, "Decoder noticed an info change (%dx%d), format=%d\n",
336 (int)mpp_frame_get_width(mppframe), (int)mpp_frame_get_height(mppframe),
337 (int)mpp_frame_get_fmt(mppframe));
339 avctx->width = mpp_frame_get_width(mppframe);
340 avctx->height = mpp_frame_get_height(mppframe);
342 decoder->mpi->control(decoder->ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);
344 av_buffer_unref(&decoder->frames_ref);
346 decoder->frames_ref = av_hwframe_ctx_alloc(decoder->device_ref);
347 if (!decoder->frames_ref) {
348 ret = AVERROR(ENOMEM);
349 goto fail;
352 mppformat = mpp_frame_get_fmt(mppframe);
353 drmformat = rkmpp_get_frameformat(mppformat);
355 hwframes = (AVHWFramesContext*)decoder->frames_ref->data;
356 hwframes->format = AV_PIX_FMT_DRM_PRIME;
357 hwframes->sw_format = drmformat == DRM_FORMAT_NV12 ? AV_PIX_FMT_NV12 : AV_PIX_FMT_NONE;
358 hwframes->width = avctx->width;
359 hwframes->height = avctx->height;
360 ret = av_hwframe_ctx_init(decoder->frames_ref);
361 if (ret < 0)
362 goto fail;
364 // here decoder is fully initialized, we need to feed it again with data
365 ret = AVERROR(EAGAIN);
366 goto fail;
367 } else if (mpp_frame_get_eos(mppframe)) {
368 av_log(avctx, AV_LOG_DEBUG, "Received a EOS frame.\n");
369 decoder->eos_reached = 1;
370 ret = AVERROR_EOF;
371 goto fail;
372 } else if (mpp_frame_get_discard(mppframe)) {
373 av_log(avctx, AV_LOG_DEBUG, "Received a discard frame.\n");
374 ret = AVERROR(EAGAIN);
375 goto fail;
376 } else if (mpp_frame_get_errinfo(mppframe)) {
377 av_log(avctx, AV_LOG_ERROR, "Received a errinfo frame.\n");
378 ret = AVERROR_UNKNOWN;
379 goto fail;
382 // here we should have a valid frame
383 av_log(avctx, AV_LOG_DEBUG, "Received a frame.\n");
385 // setup general frame fields
386 frame->format = AV_PIX_FMT_DRM_PRIME;
387 frame->width = mpp_frame_get_width(mppframe);
388 frame->height = mpp_frame_get_height(mppframe);
389 frame->pts = mpp_frame_get_pts(mppframe);
390 frame->color_range = mpp_frame_get_color_range(mppframe);
391 frame->color_primaries = mpp_frame_get_color_primaries(mppframe);
392 frame->color_trc = mpp_frame_get_color_trc(mppframe);
393 frame->colorspace = mpp_frame_get_colorspace(mppframe);
395 mode = mpp_frame_get_mode(mppframe);
396 if ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_DEINTERLACED)
397 frame->flags |= AV_FRAME_FLAG_INTERLACED;
398 if ((mode & MPP_FRAME_FLAG_FIELD_ORDER_MASK) == MPP_FRAME_FLAG_TOP_FIRST)
399 frame->flags |= AV_FRAME_FLAG_TOP_FIELD_FIRST;
401 mppformat = mpp_frame_get_fmt(mppframe);
402 drmformat = rkmpp_get_frameformat(mppformat);
404 // now setup the frame buffer info
405 buffer = mpp_frame_get_buffer(mppframe);
406 if (buffer) {
407 RKMPPFrameContext *framecontext;
408 AVDRMFrameDescriptor *desc;
409 // We allocate the descriptor in buf[0] jointly with a structure
410 // that will allow to hold additional information
411 // for properly releasing MPP frames and decoder.
412 struct {
413 AVDRMFrameDescriptor desc;
414 RKMPPFrameContext framecontext;
415 } *combined_desc = av_mallocz(sizeof(*combined_desc));
416 if (!combined_desc) {
417 ret = AVERROR(ENOMEM);
418 goto fail;
420 desc = &combined_desc->desc;
421 framecontext = &combined_desc->framecontext;
423 desc->nb_objects = 1;
424 desc->objects[0].fd = mpp_buffer_get_fd(buffer);
425 desc->objects[0].size = mpp_buffer_get_size(buffer);
427 desc->nb_layers = 1;
428 layer = &desc->layers[0];
429 layer->format = drmformat;
430 layer->nb_planes = 2;
432 layer->planes[0].object_index = 0;
433 layer->planes[0].offset = 0;
434 layer->planes[0].pitch = mpp_frame_get_hor_stride(mppframe);
436 layer->planes[1].object_index = 0;
437 layer->planes[1].offset = layer->planes[0].pitch * mpp_frame_get_ver_stride(mppframe);
438 layer->planes[1].pitch = layer->planes[0].pitch;
440 // MPP decoder needs to be closed only when all frames have been released.
441 framecontext->frame = mppframe;
443 frame->data[0] = (uint8_t *)desc;
444 frame->buf[0] = av_buffer_create((uint8_t *)desc, sizeof(*desc), rkmpp_release_frame,
445 framecontext, AV_BUFFER_FLAG_READONLY);
447 if (!frame->buf[0]) {
448 av_free(combined_desc);
449 ret = AVERROR(ENOMEM);
450 goto fail;
452 framecontext->decoder_ref = av_refstruct_ref(rk_context->decoder);
454 frame->hw_frames_ctx = av_buffer_ref(decoder->frames_ref);
455 if (!frame->hw_frames_ctx) {
456 av_frame_unref(frame);
457 return AVERROR(ENOMEM);
460 return 0;
461 } else {
462 av_log(avctx, AV_LOG_ERROR, "Failed to retrieve the frame buffer, frame is dropped (code = %d)\n", ret);
463 mpp_frame_deinit(&mppframe);
465 } else if (decoder->eos_reached) {
466 return AVERROR_EOF;
467 } else if (ret == MPP_ERR_TIMEOUT) {
468 av_log(avctx, AV_LOG_DEBUG, "Timeout when trying to get a frame from MPP\n");
471 return AVERROR(EAGAIN);
473 fail:
474 if (mppframe)
475 mpp_frame_deinit(&mppframe);
477 return ret;
480 static int rkmpp_receive_frame(AVCodecContext *avctx, AVFrame *frame)
482 RKMPPDecodeContext *rk_context = avctx->priv_data;
483 RKMPPDecoder *decoder = rk_context->decoder;
484 int ret = MPP_NOK;
485 AVPacket pkt = {0};
486 RK_S32 usedslots, freeslots;
488 if (!decoder->eos_reached) {
489 // we get the available slots in decoder
490 ret = decoder->mpi->control(decoder->ctx, MPP_DEC_GET_STREAM_COUNT, &usedslots);
491 if (ret != MPP_OK) {
492 av_log(avctx, AV_LOG_ERROR, "Failed to get decoder used slots (code = %d).\n", ret);
493 return ret;
496 freeslots = INPUT_MAX_PACKETS - usedslots;
497 if (freeslots > 0) {
498 ret = ff_decode_get_packet(avctx, &pkt);
499 if (ret < 0 && ret != AVERROR_EOF) {
500 return ret;
503 ret = rkmpp_send_packet(avctx, &pkt);
504 av_packet_unref(&pkt);
506 if (ret < 0) {
507 av_log(avctx, AV_LOG_ERROR, "Failed to send packet to decoder (code = %d)\n", ret);
508 return ret;
512 // make sure we keep decoder full
513 if (freeslots > 1)
514 return AVERROR(EAGAIN);
517 return rkmpp_retrieve_frame(avctx, frame);
520 static void rkmpp_flush(AVCodecContext *avctx)
522 RKMPPDecodeContext *rk_context = avctx->priv_data;
523 RKMPPDecoder *decoder = rk_context->decoder;
524 int ret = MPP_NOK;
526 av_log(avctx, AV_LOG_DEBUG, "Flush.\n");
528 ret = decoder->mpi->reset(decoder->ctx);
529 if (ret == MPP_OK) {
530 decoder->first_packet = 1;
531 } else
532 av_log(avctx, AV_LOG_ERROR, "Failed to reset MPI (code = %d)\n", ret);
535 static const AVCodecHWConfigInternal *const rkmpp_hw_configs[] = {
536 HW_CONFIG_INTERNAL(DRM_PRIME),
537 NULL
540 #define RKMPP_DEC_CLASS(NAME) \
541 static const AVClass rkmpp_##NAME##_dec_class = { \
542 .class_name = "rkmpp_" #NAME "_dec", \
543 .version = LIBAVUTIL_VERSION_INT, \
546 #define RKMPP_DEC(NAME, ID, BSFS) \
547 RKMPP_DEC_CLASS(NAME) \
548 const FFCodec ff_##NAME##_rkmpp_decoder = { \
549 .p.name = #NAME "_rkmpp", \
550 CODEC_LONG_NAME(#NAME " (rkmpp)"), \
551 .p.type = AVMEDIA_TYPE_VIDEO, \
552 .p.id = ID, \
553 .priv_data_size = sizeof(RKMPPDecodeContext), \
554 .init = rkmpp_init_decoder, \
555 .close = rkmpp_close_decoder, \
556 FF_CODEC_RECEIVE_FRAME_CB(rkmpp_receive_frame), \
557 .flush = rkmpp_flush, \
558 .p.priv_class = &rkmpp_##NAME##_dec_class, \
559 .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_AVOID_PROBING | AV_CODEC_CAP_HARDWARE, \
560 .hw_configs = rkmpp_hw_configs, \
561 .bsfs = BSFS, \
562 .p.wrapper_name = "rkmpp", \
563 .caps_internal = FF_CODEC_CAP_NOT_INIT_THREADSAFE, \
566 RKMPP_DEC(h264, AV_CODEC_ID_H264, "h264_mp4toannexb")
567 RKMPP_DEC(hevc, AV_CODEC_ID_HEVC, "hevc_mp4toannexb")
568 RKMPP_DEC(vp8, AV_CODEC_ID_VP8, NULL)
569 RKMPP_DEC(vp9, AV_CODEC_ID_VP9, NULL)