1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/formats/mp4/mp4_stream_parser.h"
7 #include "base/callback_helpers.h"
8 #include "base/logging.h"
9 #include "base/time/time.h"
10 #include "media/base/audio_decoder_config.h"
11 #include "media/base/stream_parser_buffer.h"
12 #include "media/base/text_track_config.h"
13 #include "media/base/video_decoder_config.h"
14 #include "media/base/video_util.h"
15 #include "media/formats/mp4/box_definitions.h"
16 #include "media/formats/mp4/box_reader.h"
17 #include "media/formats/mp4/es_descriptor.h"
18 #include "media/formats/mp4/rcheck.h"
19 #include "media/formats/mpeg/adts_constants.h"
24 MP4StreamParser::MP4StreamParser(const std::set
<int>& audio_object_types
,
26 : state_(kWaitingForInit
),
29 highest_end_offset_(0),
34 audio_object_types_(audio_object_types
),
36 is_audio_track_encrypted_(false),
37 is_video_track_encrypted_(false),
38 num_top_level_box_skipped_(0) {
41 MP4StreamParser::~MP4StreamParser() {}
43 void MP4StreamParser::Init(
44 const InitCB
& init_cb
,
45 const NewConfigCB
& config_cb
,
46 const NewBuffersCB
& new_buffers_cb
,
47 bool /* ignore_text_tracks */,
48 const EncryptedMediaInitDataCB
& encrypted_media_init_data_cb
,
49 const NewMediaSegmentCB
& new_segment_cb
,
50 const base::Closure
& end_of_segment_cb
,
51 const LogCB
& log_cb
) {
52 DCHECK_EQ(state_
, kWaitingForInit
);
53 DCHECK(init_cb_
.is_null());
54 DCHECK(!init_cb
.is_null());
55 DCHECK(!config_cb
.is_null());
56 DCHECK(!new_buffers_cb
.is_null());
57 DCHECK(!encrypted_media_init_data_cb
.is_null());
58 DCHECK(!end_of_segment_cb
.is_null());
60 ChangeState(kParsingBoxes
);
62 config_cb_
= config_cb
;
63 new_buffers_cb_
= new_buffers_cb
;
64 encrypted_media_init_data_cb_
= encrypted_media_init_data_cb
;
65 new_segment_cb_
= new_segment_cb
;
66 end_of_segment_cb_
= end_of_segment_cb
;
70 void MP4StreamParser::Reset() {
77 void MP4StreamParser::Flush() {
78 DCHECK_NE(state_
, kWaitingForInit
);
80 ChangeState(kParsingBoxes
);
83 bool MP4StreamParser::Parse(const uint8
* buf
, int size
) {
84 DCHECK_NE(state_
, kWaitingForInit
);
89 queue_
.Push(buf
, size
);
91 BufferQueue audio_buffers
;
92 BufferQueue video_buffers
;
105 result
= ParseBox(&err
);
108 case kWaitingForSampleData
:
109 result
= HaveEnoughDataToEnqueueSamples();
111 ChangeState(kEmittingSamples
);
114 case kEmittingSamples
:
115 result
= EnqueueSample(&audio_buffers
, &video_buffers
, &err
);
117 int64 max_clear
= runs_
->GetMaxClearOffset() + moof_head_
;
118 err
= !ReadAndDiscardMDATsUntil(max_clear
);
122 } while (result
&& !err
);
125 err
= !SendAndFlushSamples(&audio_buffers
, &video_buffers
);
128 DLOG(ERROR
) << "Error while parsing MP4";
138 bool MP4StreamParser::ParseBox(bool* err
) {
141 queue_
.Peek(&buf
, &size
);
142 if (!size
) return false;
144 scoped_ptr
<BoxReader
> reader(
145 BoxReader::ReadTopLevelBox(buf
, size
, log_cb_
, err
));
146 if (reader
.get() == NULL
) return false;
148 if (reader
->type() == FOURCC_MOOV
) {
149 *err
= !ParseMoov(reader
.get());
150 } else if (reader
->type() == FOURCC_MOOF
) {
151 moof_head_
= queue_
.head();
152 *err
= !ParseMoof(reader
.get());
154 // Set up first mdat offset for ReadMDATsUntil().
155 mdat_tail_
= queue_
.head() + reader
->size();
157 // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
158 // be located anywhere in the file, including inside the 'moof' itself.
159 // (Since 'default-base-is-moof' is mandated, no data references can come
160 // before the head of the 'moof', so keeping this box around is sufficient.)
163 // TODO(wolenetz,chcunningham): Enforce more strict adherence to MSE byte
164 // stream spec for ftyp and styp. See http://crbug.com/504514.
165 DVLOG(2) << "Skipping unrecognized top-level box: "
166 << FourCCToString(reader
->type());
169 queue_
.Pop(reader
->size());
173 bool MP4StreamParser::ParseMoov(BoxReader
* reader
) {
174 moov_
.reset(new Movie
);
175 RCHECK(moov_
->Parse(reader
));
181 AudioDecoderConfig audio_config
;
182 VideoDecoderConfig video_config
;
184 for (std::vector
<Track
>::const_iterator track
= moov_
->tracks
.begin();
185 track
!= moov_
->tracks
.end(); ++track
) {
186 // TODO(strobe): Only the first audio and video track present in a file are
187 // used. (Track selection is better accomplished via Source IDs, though, so
188 // adding support for track selection within a stream is low-priority.)
189 const SampleDescription
& samp_descr
=
190 track
->media
.information
.sample_table
.description
;
192 // TODO(strobe): When codec reconfigurations are supported, detect and send
193 // a codec reconfiguration for fragments using a sample description index
194 // different from the previous one
196 for (size_t t
= 0; t
< moov_
->extends
.tracks
.size(); t
++) {
197 const TrackExtends
& trex
= moov_
->extends
.tracks
[t
];
198 if (trex
.track_id
== track
->header
.track_id
) {
199 desc_idx
= trex
.default_sample_description_index
;
203 RCHECK(desc_idx
> 0);
204 desc_idx
-= 1; // BMFF descriptor index is one-based
206 if (track
->media
.handler
.type
== kAudio
&& !audio_config
.IsValidConfig()) {
207 RCHECK(!samp_descr
.audio_entries
.empty());
209 // It is not uncommon to find otherwise-valid files with incorrect sample
210 // description indices, so we fail gracefully in that case.
211 if (desc_idx
>= samp_descr
.audio_entries
.size())
213 const AudioSampleEntry
& entry
= samp_descr
.audio_entries
[desc_idx
];
214 const AAC
& aac
= entry
.esds
.aac
;
216 if (!(entry
.format
== FOURCC_MP4A
||
217 (entry
.format
== FOURCC_ENCA
&&
218 entry
.sinf
.format
.format
== FOURCC_MP4A
))) {
219 MEDIA_LOG(ERROR
, log_cb_
) << "Unsupported audio format 0x" << std::hex
220 << entry
.format
<< " in stsd box.";
224 uint8 audio_type
= entry
.esds
.object_type
;
225 DVLOG(1) << "audio_type " << std::hex
<< static_cast<int>(audio_type
);
226 if (audio_object_types_
.find(audio_type
) == audio_object_types_
.end()) {
227 MEDIA_LOG(ERROR
, log_cb_
) << "audio object type 0x" << std::hex
229 << " does not match what is specified in the"
234 AudioCodec codec
= kUnknownAudioCodec
;
235 ChannelLayout channel_layout
= CHANNEL_LAYOUT_NONE
;
236 int sample_per_second
= 0;
237 std::vector
<uint8
> extra_data
;
238 // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
239 // supported MPEG2 AAC varients.
240 if (ESDescriptor::IsAAC(audio_type
)) {
242 channel_layout
= aac
.GetChannelLayout(has_sbr_
);
243 sample_per_second
= aac
.GetOutputSamplesPerSecond(has_sbr_
);
244 #if defined(OS_ANDROID)
245 extra_data
= aac
.codec_specific_data();
248 MEDIA_LOG(ERROR
, log_cb_
) << "Unsupported audio object type 0x"
249 << std::hex
<< audio_type
<< " in esds.";
253 SampleFormat sample_format
;
254 if (entry
.samplesize
== 8) {
255 sample_format
= kSampleFormatU8
;
256 } else if (entry
.samplesize
== 16) {
257 sample_format
= kSampleFormatS16
;
258 } else if (entry
.samplesize
== 32) {
259 sample_format
= kSampleFormatS32
;
261 LOG(ERROR
) << "Unsupported sample size.";
265 is_audio_track_encrypted_
= entry
.sinf
.info
.track_encryption
.is_encrypted
;
266 DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_
;
267 audio_config
.Initialize(
268 codec
, sample_format
, channel_layout
, sample_per_second
,
269 extra_data
.size() ? &extra_data
[0] : NULL
, extra_data
.size(),
270 is_audio_track_encrypted_
, false, base::TimeDelta(),
273 audio_track_id_
= track
->header
.track_id
;
275 if (track
->media
.handler
.type
== kVideo
&& !video_config
.IsValidConfig()) {
276 RCHECK(!samp_descr
.video_entries
.empty());
277 if (desc_idx
>= samp_descr
.video_entries
.size())
279 const VideoSampleEntry
& entry
= samp_descr
.video_entries
[desc_idx
];
281 if (!entry
.IsFormatValid()) {
282 MEDIA_LOG(ERROR
, log_cb_
) << "Unsupported video format 0x" << std::hex
283 << entry
.format
<< " in stsd box.";
287 // TODO(strobe): Recover correct crop box
288 gfx::Size
coded_size(entry
.width
, entry
.height
);
289 gfx::Rect
visible_rect(coded_size
);
290 gfx::Size natural_size
= GetNaturalSize(visible_rect
.size(),
291 entry
.pixel_aspect
.h_spacing
,
292 entry
.pixel_aspect
.v_spacing
);
293 is_video_track_encrypted_
= entry
.sinf
.info
.track_encryption
.is_encrypted
;
294 DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_
;
295 video_config
.Initialize(kCodecH264
, H264PROFILE_MAIN
, VideoFrame::YV12
,
296 VideoFrame::COLOR_SPACE_UNSPECIFIED
, coded_size
,
297 visible_rect
, natural_size
,
298 // No decoder-specific buffer needed for AVC;
299 // SPS/PPS are embedded in the video stream
300 NULL
, 0, is_video_track_encrypted_
, false);
302 video_track_id_
= track
->header
.track_id
;
306 RCHECK(config_cb_
.Run(audio_config
, video_config
, TextTrackConfigMap()));
308 StreamParser::InitParameters
params(kInfiniteDuration());
309 if (moov_
->extends
.header
.fragment_duration
> 0) {
310 params
.duration
= TimeDeltaFromRational(
311 moov_
->extends
.header
.fragment_duration
, moov_
->header
.timescale
);
312 params
.liveness
= DemuxerStream::LIVENESS_RECORDED
;
313 } else if (moov_
->header
.duration
> 0 &&
314 moov_
->header
.duration
!= kuint64max
) {
316 TimeDeltaFromRational(moov_
->header
.duration
, moov_
->header
.timescale
);
317 params
.liveness
= DemuxerStream::LIVENESS_RECORDED
;
319 // In ISO/IEC 14496-12:2005(E), 8.30.2: ".. If an MP4 file is created in
320 // real-time, such as used in live streaming, it is not likely that the
321 // fragment_duration is known in advance and this (mehd) box may be
323 // TODO(wolenetz): Investigate gating liveness detection on timeline_offset
324 // when it's populated. See http://crbug.com/312699
325 params
.liveness
= DemuxerStream::LIVENESS_LIVE
;
328 DVLOG(1) << "liveness: " << params
.liveness
;
330 if (!init_cb_
.is_null())
331 base::ResetAndReturn(&init_cb_
).Run(params
);
333 if (!moov_
->pssh
.empty())
334 OnEncryptedMediaInitData(moov_
->pssh
);
339 bool MP4StreamParser::ParseMoof(BoxReader
* reader
) {
340 RCHECK(moov_
.get()); // Must already have initialization segment
342 RCHECK(moof
.Parse(reader
));
344 runs_
.reset(new TrackRunIterator(moov_
.get(), log_cb_
));
345 RCHECK(runs_
->Init(moof
));
346 RCHECK(ComputeHighestEndOffset(moof
));
348 if (!moof
.pssh
.empty())
349 OnEncryptedMediaInitData(moof
.pssh
);
351 new_segment_cb_
.Run();
352 ChangeState(kWaitingForSampleData
);
356 void MP4StreamParser::OnEncryptedMediaInitData(
357 const std::vector
<ProtectionSystemSpecificHeader
>& headers
) {
358 // TODO(strobe): ensure that the value of init_data (all PSSH headers
359 // concatenated in arbitrary order) matches the EME spec.
360 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
361 size_t total_size
= 0;
362 for (size_t i
= 0; i
< headers
.size(); i
++)
363 total_size
+= headers
[i
].raw_box
.size();
365 std::vector
<uint8
> init_data(total_size
);
367 for (size_t i
= 0; i
< headers
.size(); i
++) {
368 memcpy(&init_data
[pos
], &headers
[i
].raw_box
[0],
369 headers
[i
].raw_box
.size());
370 pos
+= headers
[i
].raw_box
.size();
372 encrypted_media_init_data_cb_
.Run(EmeInitDataType::CENC
, init_data
);
375 bool MP4StreamParser::PrepareAVCBuffer(
376 const AVCDecoderConfigurationRecord
& avc_config
,
377 std::vector
<uint8
>* frame_buf
,
378 std::vector
<SubsampleEntry
>* subsamples
) const {
379 // Convert the AVC NALU length fields to Annex B headers, as expected by
380 // decoding libraries. Since this may enlarge the size of the buffer, we also
381 // update the clear byte count for each subsample if encryption is used to
382 // account for the difference in size between the length prefix and Annex B
384 RCHECK(AVC::ConvertFrameToAnnexB(avc_config
.length_size
, frame_buf
));
385 if (!subsamples
->empty()) {
386 const int nalu_size_diff
= 4 - avc_config
.length_size
;
387 size_t expected_size
= runs_
->sample_size() +
388 subsamples
->size() * nalu_size_diff
;
389 RCHECK(frame_buf
->size() == expected_size
);
390 for (size_t i
= 0; i
< subsamples
->size(); i
++)
391 (*subsamples
)[i
].clear_bytes
+= nalu_size_diff
;
394 if (runs_
->is_keyframe()) {
395 // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
396 // a frame. If subsample info is present, we also update the clear byte
397 // count for that first subsample.
398 RCHECK(AVC::InsertParamSetsAnnexB(avc_config
, frame_buf
, subsamples
));
401 DCHECK(AVC::IsValidAnnexB(*frame_buf
, *subsamples
));
405 bool MP4StreamParser::PrepareAACBuffer(
406 const AAC
& aac_config
, std::vector
<uint8
>* frame_buf
,
407 std::vector
<SubsampleEntry
>* subsamples
) const {
408 // Append an ADTS header to every audio sample.
409 RCHECK(aac_config
.ConvertEsdsToADTS(frame_buf
));
411 // As above, adjust subsample information to account for the headers. AAC is
412 // not required to use subsample encryption, so we may need to add an entry.
413 if (subsamples
->empty()) {
414 subsamples
->push_back(SubsampleEntry(
415 kADTSHeaderMinSize
, frame_buf
->size() - kADTSHeaderMinSize
));
417 (*subsamples
)[0].clear_bytes
+= kADTSHeaderMinSize
;
422 bool MP4StreamParser::EnqueueSample(BufferQueue
* audio_buffers
,
423 BufferQueue
* video_buffers
,
425 DCHECK_EQ(state_
, kEmittingSamples
);
427 if (!runs_
->IsRunValid()) {
428 // Flush any buffers we've gotten in this chunk so that buffers don't
429 // cross NewSegment() calls
430 *err
= !SendAndFlushSamples(audio_buffers
, video_buffers
);
434 // Remain in kEmittingSamples state, discarding data, until the end of
435 // the current 'mdat' box has been appended to the queue.
436 if (!queue_
.Trim(mdat_tail_
))
439 ChangeState(kParsingBoxes
);
440 end_of_segment_cb_
.Run();
444 if (!runs_
->IsSampleValid()) {
453 queue_
.Peek(&buf
, &buf_size
);
454 if (!buf_size
) return false;
456 bool audio
= has_audio_
&& audio_track_id_
== runs_
->track_id();
457 bool video
= has_video_
&& video_track_id_
== runs_
->track_id();
459 // Skip this entire track if it's not one we're interested in
460 if (!audio
&& !video
) {
465 // Attempt to cache the auxiliary information first. Aux info is usually
466 // placed in a contiguous block before the sample data, rather than being
467 // interleaved. If we didn't cache it, this would require that we retain the
468 // start of the segment buffer while reading samples. Aux info is typically
469 // quite small compared to sample data, so this pattern is useful on
470 // memory-constrained devices where the source buffer consumes a substantial
471 // portion of the total system memory.
472 if (runs_
->AuxInfoNeedsToBeCached()) {
473 queue_
.PeekAt(runs_
->aux_info_offset() + moof_head_
, &buf
, &buf_size
);
474 if (buf_size
< runs_
->aux_info_size()) return false;
475 *err
= !runs_
->CacheAuxInfo(buf
, buf_size
);
479 queue_
.PeekAt(runs_
->sample_offset() + moof_head_
, &buf
, &buf_size
);
480 if (buf_size
< runs_
->sample_size()) return false;
482 scoped_ptr
<DecryptConfig
> decrypt_config
;
483 std::vector
<SubsampleEntry
> subsamples
;
484 if (runs_
->is_encrypted()) {
485 decrypt_config
= runs_
->GetDecryptConfig();
486 if (!decrypt_config
) {
490 subsamples
= decrypt_config
->subsamples();
493 std::vector
<uint8
> frame_buf(buf
, buf
+ runs_
->sample_size());
495 if (!PrepareAVCBuffer(runs_
->video_description().avcc
,
496 &frame_buf
, &subsamples
)) {
497 MEDIA_LOG(ERROR
, log_cb_
) << "Failed to prepare AVC sample for decode";
504 if (ESDescriptor::IsAAC(runs_
->audio_description().esds
.object_type
) &&
505 !PrepareAACBuffer(runs_
->audio_description().esds
.aac
,
506 &frame_buf
, &subsamples
)) {
507 MEDIA_LOG(ERROR
, log_cb_
) << "Failed to prepare AAC sample for decode";
513 if (decrypt_config
) {
514 if (!subsamples
.empty()) {
515 // Create a new config with the updated subsamples.
516 decrypt_config
.reset(new DecryptConfig(
517 decrypt_config
->key_id(),
518 decrypt_config
->iv(),
521 // else, use the existing config.
522 } else if ((audio
&& is_audio_track_encrypted_
) ||
523 (video
&& is_video_track_encrypted_
)) {
524 // The media pipeline requires a DecryptConfig with an empty |iv|.
525 // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
526 decrypt_config
.reset(
527 new DecryptConfig("1", "", std::vector
<SubsampleEntry
>()));
530 StreamParserBuffer::Type buffer_type
= audio
? DemuxerStream::AUDIO
:
531 DemuxerStream::VIDEO
;
533 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId
534 // type and allow multiple tracks for same media type, if applicable. See
535 // https://crbug.com/341581.
537 // NOTE: MPEG's "random access point" concept is equivalent to the
538 // downstream code's "is keyframe" concept.
539 scoped_refptr
<StreamParserBuffer
> stream_buf
=
540 StreamParserBuffer::CopyFrom(&frame_buf
[0], frame_buf
.size(),
541 runs_
->is_random_access_point(),
545 stream_buf
->set_decrypt_config(decrypt_config
.Pass());
547 stream_buf
->set_duration(runs_
->duration());
548 stream_buf
->set_timestamp(runs_
->cts());
549 stream_buf
->SetDecodeTimestamp(runs_
->dts());
551 DVLOG(3) << "Pushing frame: aud=" << audio
552 << ", key=" << runs_
->is_keyframe()
553 << ", rap=" << runs_
->is_random_access_point()
554 << ", dur=" << runs_
->duration().InMilliseconds()
555 << ", dts=" << runs_
->dts().InMilliseconds()
556 << ", cts=" << runs_
->cts().InMilliseconds()
557 << ", size=" << runs_
->sample_size();
560 audio_buffers
->push_back(stream_buf
);
562 video_buffers
->push_back(stream_buf
);
565 runs_
->AdvanceSample();
569 bool MP4StreamParser::SendAndFlushSamples(BufferQueue
* audio_buffers
,
570 BufferQueue
* video_buffers
) {
571 if (audio_buffers
->empty() && video_buffers
->empty())
574 TextBufferQueueMap empty_text_map
;
575 bool success
= new_buffers_cb_
.Run(*audio_buffers
,
578 audio_buffers
->clear();
579 video_buffers
->clear();
583 bool MP4StreamParser::ReadAndDiscardMDATsUntil(int64 max_clear_offset
) {
585 int64 upper_bound
= std::min(max_clear_offset
, queue_
.tail());
586 while (mdat_tail_
< upper_bound
) {
587 const uint8
* buf
= NULL
;
589 queue_
.PeekAt(mdat_tail_
, &buf
, &size
);
593 if (!BoxReader::StartTopLevelBox(buf
, size
, log_cb_
,
594 &type
, &box_sz
, &err
))
597 if (type
!= FOURCC_MDAT
) {
598 MEDIA_LOG(DEBUG
, log_cb_
) << "Unexpected box type while parsing MDATs: "
599 << FourCCToString(type
);
601 mdat_tail_
+= box_sz
;
603 queue_
.Trim(std::min(mdat_tail_
, upper_bound
));
607 void MP4StreamParser::ChangeState(State new_state
) {
608 DVLOG(2) << "Changing state: " << new_state
;
612 bool MP4StreamParser::HaveEnoughDataToEnqueueSamples() {
613 DCHECK_EQ(state_
, kWaitingForSampleData
);
614 // For muxed content, make sure we have data up to |highest_end_offset_|
615 // so we can ensure proper enqueuing behavior. Otherwise assume we have enough
616 // data and allow per sample offset checks to meter sample enqueuing.
617 // TODO(acolwell): Fix trun box handling so we don't have to special case
619 return !(has_audio_
&& has_video_
&&
620 queue_
.tail() < highest_end_offset_
+ moof_head_
);
623 bool MP4StreamParser::ComputeHighestEndOffset(const MovieFragment
& moof
) {
624 highest_end_offset_
= 0;
626 TrackRunIterator
runs(moov_
.get(), log_cb_
);
627 RCHECK(runs
.Init(moof
));
629 while (runs
.IsRunValid()) {
630 int64 aux_info_end_offset
= runs
.aux_info_offset() + runs
.aux_info_size();
631 if (aux_info_end_offset
> highest_end_offset_
)
632 highest_end_offset_
= aux_info_end_offset
;
634 while (runs
.IsSampleValid()) {
635 int64 sample_end_offset
= runs
.sample_offset() + runs
.sample_size();
636 if (sample_end_offset
> highest_end_offset_
)
637 highest_end_offset_
= sample_end_offset
;
639 runs
.AdvanceSample();