1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/ffmpeg/ffmpeg_common.h"
7 #include "base/basictypes.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_number_conversions.h"
11 #include "media/base/decoder_buffer.h"
12 #include "media/base/video_frame.h"
13 #include "media/base/video_util.h"
17 // Why FF_INPUT_BUFFER_PADDING_SIZE? FFmpeg assumes all input buffers are
18 // padded. Check here to ensure FFmpeg only receives data padded to its
20 COMPILE_ASSERT(DecoderBuffer::kPaddingSize
>= FF_INPUT_BUFFER_PADDING_SIZE
,
21 decoder_buffer_padding_size_does_not_fit_ffmpeg_requirement
);
23 // Alignment requirement by FFmpeg for input and output buffers. This need to
24 // be updated to match FFmpeg when it changes.
25 #if defined(ARCH_CPU_ARM_FAMILY)
26 static const int kFFmpegBufferAddressAlignment
= 16;
28 static const int kFFmpegBufferAddressAlignment
= 32;
31 // Check here to ensure FFmpeg only receives data aligned to its specifications.
33 DecoderBuffer::kAlignmentSize
>= kFFmpegBufferAddressAlignment
&&
34 DecoderBuffer::kAlignmentSize
% kFFmpegBufferAddressAlignment
== 0,
35 decoder_buffer_alignment_size_does_not_fit_ffmpeg_requirement
);
37 // Allows faster SIMD YUV convert. Also, FFmpeg overreads/-writes occasionally.
38 // See video_get_buffer() in libavcodec/utils.c.
39 static const int kFFmpegOutputBufferPaddingSize
= 16;
41 COMPILE_ASSERT(VideoFrame::kFrameSizePadding
>= kFFmpegOutputBufferPaddingSize
,
42 video_frame_padding_size_does_not_fit_ffmpeg_requirement
);
45 VideoFrame::kFrameAddressAlignment
>= kFFmpegBufferAddressAlignment
&&
46 VideoFrame::kFrameAddressAlignment
% kFFmpegBufferAddressAlignment
== 0,
47 video_frame_address_alignment_does_not_fit_ffmpeg_requirement
);
49 static const AVRational kMicrosBase
= { 1, base::Time::kMicrosecondsPerSecond
};
51 base::TimeDelta
ConvertFromTimeBase(const AVRational
& time_base
,
53 int64 microseconds
= av_rescale_q(timestamp
, time_base
, kMicrosBase
);
54 return base::TimeDelta::FromMicroseconds(microseconds
);
57 int64
ConvertToTimeBase(const AVRational
& time_base
,
58 const base::TimeDelta
& timestamp
) {
59 return av_rescale_q(timestamp
.InMicroseconds(), kMicrosBase
, time_base
);
62 // Converts an FFmpeg audio codec ID into its corresponding supported codec id.
63 static AudioCodec
CodecIDToAudioCodec(AVCodecID codec_id
) {
69 case AV_CODEC_ID_VORBIS
:
71 case AV_CODEC_ID_PCM_U8
:
72 case AV_CODEC_ID_PCM_S16LE
:
73 case AV_CODEC_ID_PCM_S24LE
:
74 case AV_CODEC_ID_PCM_F32LE
:
76 case AV_CODEC_ID_PCM_S16BE
:
77 return kCodecPCM_S16BE
;
78 case AV_CODEC_ID_PCM_S24BE
:
79 return kCodecPCM_S24BE
;
80 case AV_CODEC_ID_FLAC
:
82 case AV_CODEC_ID_AMR_NB
:
84 case AV_CODEC_ID_AMR_WB
:
86 case AV_CODEC_ID_GSM_MS
:
88 case AV_CODEC_ID_PCM_ALAW
:
89 return kCodecPCM_ALAW
;
90 case AV_CODEC_ID_PCM_MULAW
:
91 return kCodecPCM_MULAW
;
92 case AV_CODEC_ID_OPUS
:
95 DVLOG(1) << "Unknown audio CodecID: " << codec_id
;
97 return kUnknownAudioCodec
;
100 static AVCodecID
AudioCodecToCodecID(AudioCodec audio_codec
,
101 SampleFormat sample_format
) {
102 switch (audio_codec
) {
104 return AV_CODEC_ID_AAC
;
106 return AV_CODEC_ID_MP3
;
108 switch (sample_format
) {
109 case kSampleFormatU8
:
110 return AV_CODEC_ID_PCM_U8
;
111 case kSampleFormatS16
:
112 return AV_CODEC_ID_PCM_S16LE
;
113 case kSampleFormatS32
:
114 return AV_CODEC_ID_PCM_S24LE
;
115 case kSampleFormatF32
:
116 return AV_CODEC_ID_PCM_F32LE
;
118 DVLOG(1) << "Unsupported sample format: " << sample_format
;
121 case kCodecPCM_S16BE
:
122 return AV_CODEC_ID_PCM_S16BE
;
123 case kCodecPCM_S24BE
:
124 return AV_CODEC_ID_PCM_S24BE
;
126 return AV_CODEC_ID_VORBIS
;
128 return AV_CODEC_ID_FLAC
;
130 return AV_CODEC_ID_AMR_NB
;
132 return AV_CODEC_ID_AMR_WB
;
134 return AV_CODEC_ID_GSM_MS
;
136 return AV_CODEC_ID_PCM_ALAW
;
137 case kCodecPCM_MULAW
:
138 return AV_CODEC_ID_PCM_MULAW
;
140 return AV_CODEC_ID_OPUS
;
142 DVLOG(1) << "Unknown AudioCodec: " << audio_codec
;
144 return AV_CODEC_ID_NONE
;
147 // Converts an FFmpeg video codec ID into its corresponding supported codec id.
148 static VideoCodec
CodecIDToVideoCodec(AVCodecID codec_id
) {
150 case AV_CODEC_ID_H264
:
152 case AV_CODEC_ID_THEORA
:
154 case AV_CODEC_ID_MPEG4
:
156 case AV_CODEC_ID_VP8
:
158 case AV_CODEC_ID_VP9
:
161 DVLOG(1) << "Unknown video CodecID: " << codec_id
;
163 return kUnknownVideoCodec
;
166 static AVCodecID
VideoCodecToCodecID(VideoCodec video_codec
) {
167 switch (video_codec
) {
169 return AV_CODEC_ID_H264
;
171 return AV_CODEC_ID_THEORA
;
173 return AV_CODEC_ID_MPEG4
;
175 return AV_CODEC_ID_VP8
;
177 return AV_CODEC_ID_VP9
;
179 DVLOG(1) << "Unknown VideoCodec: " << video_codec
;
181 return AV_CODEC_ID_NONE
;
184 static VideoCodecProfile
ProfileIDToVideoCodecProfile(int profile
) {
185 // Clear out the CONSTRAINED & INTRA flags which are strict subsets of the
186 // corresponding profiles with which they're used.
187 profile
&= ~FF_PROFILE_H264_CONSTRAINED
;
188 profile
&= ~FF_PROFILE_H264_INTRA
;
190 case FF_PROFILE_H264_BASELINE
:
191 return H264PROFILE_BASELINE
;
192 case FF_PROFILE_H264_MAIN
:
193 return H264PROFILE_MAIN
;
194 case FF_PROFILE_H264_EXTENDED
:
195 return H264PROFILE_EXTENDED
;
196 case FF_PROFILE_H264_HIGH
:
197 return H264PROFILE_HIGH
;
198 case FF_PROFILE_H264_HIGH_10
:
199 return H264PROFILE_HIGH10PROFILE
;
200 case FF_PROFILE_H264_HIGH_422
:
201 return H264PROFILE_HIGH422PROFILE
;
202 case FF_PROFILE_H264_HIGH_444_PREDICTIVE
:
203 return H264PROFILE_HIGH444PREDICTIVEPROFILE
;
205 DVLOG(1) << "Unknown profile id: " << profile
;
207 return VIDEO_CODEC_PROFILE_UNKNOWN
;
210 static int VideoCodecProfileToProfileID(VideoCodecProfile profile
) {
212 case H264PROFILE_BASELINE
:
213 return FF_PROFILE_H264_BASELINE
;
214 case H264PROFILE_MAIN
:
215 return FF_PROFILE_H264_MAIN
;
216 case H264PROFILE_EXTENDED
:
217 return FF_PROFILE_H264_EXTENDED
;
218 case H264PROFILE_HIGH
:
219 return FF_PROFILE_H264_HIGH
;
220 case H264PROFILE_HIGH10PROFILE
:
221 return FF_PROFILE_H264_HIGH_10
;
222 case H264PROFILE_HIGH422PROFILE
:
223 return FF_PROFILE_H264_HIGH_422
;
224 case H264PROFILE_HIGH444PREDICTIVEPROFILE
:
225 return FF_PROFILE_H264_HIGH_444_PREDICTIVE
;
227 DVLOG(1) << "Unknown VideoCodecProfile: " << profile
;
229 return FF_PROFILE_UNKNOWN
;
232 SampleFormat
AVSampleFormatToSampleFormat(AVSampleFormat sample_format
) {
233 switch (sample_format
) {
234 case AV_SAMPLE_FMT_U8
:
235 return kSampleFormatU8
;
236 case AV_SAMPLE_FMT_S16
:
237 return kSampleFormatS16
;
238 case AV_SAMPLE_FMT_S32
:
239 return kSampleFormatS32
;
240 case AV_SAMPLE_FMT_FLT
:
241 return kSampleFormatF32
;
242 case AV_SAMPLE_FMT_S16P
:
243 return kSampleFormatPlanarS16
;
244 case AV_SAMPLE_FMT_FLTP
:
245 return kSampleFormatPlanarF32
;
247 DVLOG(1) << "Unknown AVSampleFormat: " << sample_format
;
249 return kUnknownSampleFormat
;
252 static AVSampleFormat
SampleFormatToAVSampleFormat(SampleFormat sample_format
) {
253 switch (sample_format
) {
254 case kSampleFormatU8
:
255 return AV_SAMPLE_FMT_U8
;
256 case kSampleFormatS16
:
257 return AV_SAMPLE_FMT_S16
;
258 case kSampleFormatS32
:
259 return AV_SAMPLE_FMT_S32
;
260 case kSampleFormatF32
:
261 return AV_SAMPLE_FMT_FLT
;
262 case kSampleFormatPlanarS16
:
263 return AV_SAMPLE_FMT_S16P
;
264 case kSampleFormatPlanarF32
:
265 return AV_SAMPLE_FMT_FLTP
;
267 DVLOG(1) << "Unknown SampleFormat: " << sample_format
;
269 return AV_SAMPLE_FMT_NONE
;
272 static void AVCodecContextToAudioDecoderConfig(
273 const AVCodecContext
* codec_context
,
275 AudioDecoderConfig
* config
,
277 DCHECK_EQ(codec_context
->codec_type
, AVMEDIA_TYPE_AUDIO
);
279 AudioCodec codec
= CodecIDToAudioCodec(codec_context
->codec_id
);
281 SampleFormat sample_format
=
282 AVSampleFormatToSampleFormat(codec_context
->sample_fmt
);
284 ChannelLayout channel_layout
= ChannelLayoutToChromeChannelLayout(
285 codec_context
->channel_layout
, codec_context
->channels
);
287 if (codec
== kCodecOpus
) {
288 // |codec_context->sample_fmt| is not set by FFmpeg because Opus decoding is
289 // not enabled in FFmpeg. It doesn't matter what value is set here, so long
290 // as it's valid, the true sample format is selected inside the decoder.
291 sample_format
= kSampleFormatF32
;
294 base::TimeDelta seek_preroll
;
295 if (codec_context
->seek_preroll
> 0) {
296 seek_preroll
= base::TimeDelta::FromMicroseconds(
297 codec_context
->seek_preroll
* 1000000.0 / codec_context
->sample_rate
);
300 base::TimeDelta codec_delay
;
301 if (codec_context
->delay
> 0) {
302 codec_delay
= base::TimeDelta::FromMicroseconds(
303 codec_context
->delay
* 1000000.0 / codec_context
->sample_rate
);
306 config
->Initialize(codec
,
309 codec_context
->sample_rate
,
310 codec_context
->extradata
,
311 codec_context
->extradata_size
,
316 if (codec
!= kCodecOpus
) {
317 DCHECK_EQ(av_get_bytes_per_sample(codec_context
->sample_fmt
) * 8,
318 config
->bits_per_channel());
322 void AVStreamToAudioDecoderConfig(
323 const AVStream
* stream
,
324 AudioDecoderConfig
* config
,
326 bool is_encrypted
= false;
327 AVDictionaryEntry
* key
= av_dict_get(stream
->metadata
, "enc_key_id", NULL
, 0);
330 return AVCodecContextToAudioDecoderConfig(
331 stream
->codec
, is_encrypted
, config
, record_stats
);
334 void AudioDecoderConfigToAVCodecContext(const AudioDecoderConfig
& config
,
335 AVCodecContext
* codec_context
) {
336 codec_context
->codec_type
= AVMEDIA_TYPE_AUDIO
;
337 codec_context
->codec_id
= AudioCodecToCodecID(config
.codec(),
338 config
.sample_format());
339 codec_context
->sample_fmt
= SampleFormatToAVSampleFormat(
340 config
.sample_format());
342 // TODO(scherkus): should we set |channel_layout|? I'm not sure if FFmpeg uses
343 // said information to decode.
344 codec_context
->channels
=
345 ChannelLayoutToChannelCount(config
.channel_layout());
346 codec_context
->sample_rate
= config
.samples_per_second();
348 if (config
.extra_data()) {
349 codec_context
->extradata_size
= config
.extra_data_size();
350 codec_context
->extradata
= reinterpret_cast<uint8_t*>(
351 av_malloc(config
.extra_data_size() + FF_INPUT_BUFFER_PADDING_SIZE
));
352 memcpy(codec_context
->extradata
, config
.extra_data(),
353 config
.extra_data_size());
354 memset(codec_context
->extradata
+ config
.extra_data_size(), '\0',
355 FF_INPUT_BUFFER_PADDING_SIZE
);
357 codec_context
->extradata
= NULL
;
358 codec_context
->extradata_size
= 0;
362 void AVStreamToVideoDecoderConfig(
363 const AVStream
* stream
,
364 VideoDecoderConfig
* config
,
366 gfx::Size
coded_size(stream
->codec
->coded_width
, stream
->codec
->coded_height
);
368 // TODO(vrk): This assumes decoded frame data starts at (0, 0), which is true
369 // for now, but may not always be true forever. Fix this in the future.
370 gfx::Rect
visible_rect(stream
->codec
->width
, stream
->codec
->height
);
372 AVRational aspect_ratio
= { 1, 1 };
373 if (stream
->sample_aspect_ratio
.num
)
374 aspect_ratio
= stream
->sample_aspect_ratio
;
375 else if (stream
->codec
->sample_aspect_ratio
.num
)
376 aspect_ratio
= stream
->codec
->sample_aspect_ratio
;
378 VideoCodec codec
= CodecIDToVideoCodec(stream
->codec
->codec_id
);
380 VideoCodecProfile profile
= VIDEO_CODEC_PROFILE_UNKNOWN
;
381 if (codec
== kCodecVP8
)
382 profile
= VP8PROFILE_MAIN
;
383 else if (codec
== kCodecVP9
)
384 profile
= VP9PROFILE_MAIN
;
386 profile
= ProfileIDToVideoCodecProfile(stream
->codec
->profile
);
388 gfx::Size natural_size
= GetNaturalSize(
389 visible_rect
.size(), aspect_ratio
.num
, aspect_ratio
.den
);
392 UMA_HISTOGRAM_ENUMERATION("Media.VideoColorRange",
393 stream
->codec
->color_range
,
397 VideoFrame::Format format
= PixelFormatToVideoFormat(stream
->codec
->pix_fmt
);
398 if (codec
== kCodecVP9
) {
399 // TODO(tomfinegan): libavcodec doesn't know about VP9.
400 format
= VideoFrame::YV12
;
401 coded_size
= natural_size
;
404 bool is_encrypted
= false;
405 AVDictionaryEntry
* key
= av_dict_get(stream
->metadata
, "enc_key_id", NULL
, 0);
409 AVDictionaryEntry
* webm_alpha
=
410 av_dict_get(stream
->metadata
, "alpha_mode", NULL
, 0);
411 if (webm_alpha
&& !strcmp(webm_alpha
->value
, "1")) {
412 format
= VideoFrame::YV12A
;
415 config
->Initialize(codec
,
418 coded_size
, visible_rect
, natural_size
,
419 stream
->codec
->extradata
, stream
->codec
->extradata_size
,
424 void VideoDecoderConfigToAVCodecContext(
425 const VideoDecoderConfig
& config
,
426 AVCodecContext
* codec_context
) {
427 codec_context
->codec_type
= AVMEDIA_TYPE_VIDEO
;
428 codec_context
->codec_id
= VideoCodecToCodecID(config
.codec());
429 codec_context
->profile
= VideoCodecProfileToProfileID(config
.profile());
430 codec_context
->coded_width
= config
.coded_size().width();
431 codec_context
->coded_height
= config
.coded_size().height();
432 codec_context
->pix_fmt
= VideoFormatToPixelFormat(config
.format());
434 if (config
.extra_data()) {
435 codec_context
->extradata_size
= config
.extra_data_size();
436 codec_context
->extradata
= reinterpret_cast<uint8_t*>(
437 av_malloc(config
.extra_data_size() + FF_INPUT_BUFFER_PADDING_SIZE
));
438 memcpy(codec_context
->extradata
, config
.extra_data(),
439 config
.extra_data_size());
440 memset(codec_context
->extradata
+ config
.extra_data_size(), '\0',
441 FF_INPUT_BUFFER_PADDING_SIZE
);
443 codec_context
->extradata
= NULL
;
444 codec_context
->extradata_size
= 0;
448 ChannelLayout
ChannelLayoutToChromeChannelLayout(int64_t layout
, int channels
) {
450 case AV_CH_LAYOUT_MONO
:
451 return CHANNEL_LAYOUT_MONO
;
452 case AV_CH_LAYOUT_STEREO
:
453 return CHANNEL_LAYOUT_STEREO
;
454 case AV_CH_LAYOUT_2_1
:
455 return CHANNEL_LAYOUT_2_1
;
456 case AV_CH_LAYOUT_SURROUND
:
457 return CHANNEL_LAYOUT_SURROUND
;
458 case AV_CH_LAYOUT_4POINT0
:
459 return CHANNEL_LAYOUT_4_0
;
460 case AV_CH_LAYOUT_2_2
:
461 return CHANNEL_LAYOUT_2_2
;
462 case AV_CH_LAYOUT_QUAD
:
463 return CHANNEL_LAYOUT_QUAD
;
464 case AV_CH_LAYOUT_5POINT0
:
465 return CHANNEL_LAYOUT_5_0
;
466 case AV_CH_LAYOUT_5POINT1
:
467 return CHANNEL_LAYOUT_5_1
;
468 case AV_CH_LAYOUT_5POINT0_BACK
:
469 return CHANNEL_LAYOUT_5_0_BACK
;
470 case AV_CH_LAYOUT_5POINT1_BACK
:
471 return CHANNEL_LAYOUT_5_1_BACK
;
472 case AV_CH_LAYOUT_7POINT0
:
473 return CHANNEL_LAYOUT_7_0
;
474 case AV_CH_LAYOUT_7POINT1
:
475 return CHANNEL_LAYOUT_7_1
;
476 case AV_CH_LAYOUT_7POINT1_WIDE
:
477 return CHANNEL_LAYOUT_7_1_WIDE
;
478 case AV_CH_LAYOUT_STEREO_DOWNMIX
:
479 return CHANNEL_LAYOUT_STEREO_DOWNMIX
;
480 case AV_CH_LAYOUT_2POINT1
:
481 return CHANNEL_LAYOUT_2POINT1
;
482 case AV_CH_LAYOUT_3POINT1
:
483 return CHANNEL_LAYOUT_3_1
;
484 case AV_CH_LAYOUT_4POINT1
:
485 return CHANNEL_LAYOUT_4_1
;
486 case AV_CH_LAYOUT_6POINT0
:
487 return CHANNEL_LAYOUT_6_0
;
488 case AV_CH_LAYOUT_6POINT0_FRONT
:
489 return CHANNEL_LAYOUT_6_0_FRONT
;
490 case AV_CH_LAYOUT_HEXAGONAL
:
491 return CHANNEL_LAYOUT_HEXAGONAL
;
492 case AV_CH_LAYOUT_6POINT1
:
493 return CHANNEL_LAYOUT_6_1
;
494 case AV_CH_LAYOUT_6POINT1_BACK
:
495 return CHANNEL_LAYOUT_6_1_BACK
;
496 case AV_CH_LAYOUT_6POINT1_FRONT
:
497 return CHANNEL_LAYOUT_6_1_FRONT
;
498 case AV_CH_LAYOUT_7POINT0_FRONT
:
499 return CHANNEL_LAYOUT_7_0_FRONT
;
500 #ifdef AV_CH_LAYOUT_7POINT1_WIDE_BACK
501 case AV_CH_LAYOUT_7POINT1_WIDE_BACK
:
502 return CHANNEL_LAYOUT_7_1_WIDE_BACK
;
504 case AV_CH_LAYOUT_OCTAGONAL
:
505 return CHANNEL_LAYOUT_OCTAGONAL
;
507 // FFmpeg channel_layout is 0 for .wav and .mp3. Attempt to guess layout
508 // based on the channel count.
509 return GuessChannelLayout(channels
);
513 VideoFrame::Format
PixelFormatToVideoFormat(PixelFormat pixel_format
) {
514 switch (pixel_format
) {
515 case PIX_FMT_YUV422P
:
516 return VideoFrame::YV16
;
517 case PIX_FMT_YUV420P
:
518 return VideoFrame::YV12
;
519 case PIX_FMT_YUVJ420P
:
520 return VideoFrame::YV12J
;
521 case PIX_FMT_YUVA420P
:
522 return VideoFrame::YV12A
;
524 DVLOG(1) << "Unsupported PixelFormat: " << pixel_format
;
526 return VideoFrame::UNKNOWN
;
529 PixelFormat
VideoFormatToPixelFormat(VideoFrame::Format video_format
) {
530 switch (video_format
) {
531 case VideoFrame::YV16
:
532 return PIX_FMT_YUV422P
;
533 case VideoFrame::YV12
:
534 return PIX_FMT_YUV420P
;
535 case VideoFrame::YV12J
:
536 return PIX_FMT_YUVJ420P
;
537 case VideoFrame::YV12A
:
538 return PIX_FMT_YUVA420P
;
540 DVLOG(1) << "Unsupported VideoFrame::Format: " << video_format
;