1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/formats/mp4/mp4_stream_parser.h"
7 #include "base/callback.h"
8 #include "base/callback_helpers.h"
9 #include "base/logging.h"
10 #include "base/time/time.h"
11 #include "media/base/audio_decoder_config.h"
12 #include "media/base/stream_parser_buffer.h"
13 #include "media/base/text_track_config.h"
14 #include "media/base/video_decoder_config.h"
15 #include "media/base/video_util.h"
16 #include "media/formats/mp4/box_definitions.h"
17 #include "media/formats/mp4/box_reader.h"
18 #include "media/formats/mp4/es_descriptor.h"
19 #include "media/formats/mp4/rcheck.h"
20 #include "media/formats/mpeg/adts_constants.h"
25 MP4StreamParser::MP4StreamParser(const std::set
<int>& audio_object_types
,
27 : state_(kWaitingForInit
),
30 highest_end_offset_(0),
35 audio_object_types_(audio_object_types
),
37 is_audio_track_encrypted_(false),
38 is_video_track_encrypted_(false) {
41 MP4StreamParser::~MP4StreamParser() {}
43 void MP4StreamParser::Init(
44 const InitCB
& init_cb
,
45 const NewConfigCB
& config_cb
,
46 const NewBuffersCB
& new_buffers_cb
,
47 bool /* ignore_text_tracks */,
48 const EncryptedMediaInitDataCB
& encrypted_media_init_data_cb
,
49 const NewMediaSegmentCB
& new_segment_cb
,
50 const base::Closure
& end_of_segment_cb
,
51 const LogCB
& log_cb
) {
52 DCHECK_EQ(state_
, kWaitingForInit
);
53 DCHECK(init_cb_
.is_null());
54 DCHECK(!init_cb
.is_null());
55 DCHECK(!config_cb
.is_null());
56 DCHECK(!new_buffers_cb
.is_null());
57 DCHECK(!encrypted_media_init_data_cb
.is_null());
58 DCHECK(!end_of_segment_cb
.is_null());
60 ChangeState(kParsingBoxes
);
62 config_cb_
= config_cb
;
63 new_buffers_cb_
= new_buffers_cb
;
64 encrypted_media_init_data_cb_
= encrypted_media_init_data_cb
;
65 new_segment_cb_
= new_segment_cb
;
66 end_of_segment_cb_
= end_of_segment_cb
;
70 void MP4StreamParser::Reset() {
77 void MP4StreamParser::Flush() {
78 DCHECK_NE(state_
, kWaitingForInit
);
80 ChangeState(kParsingBoxes
);
83 bool MP4StreamParser::Parse(const uint8
* buf
, int size
) {
84 DCHECK_NE(state_
, kWaitingForInit
);
89 queue_
.Push(buf
, size
);
91 BufferQueue audio_buffers
;
92 BufferQueue video_buffers
;
105 result
= ParseBox(&err
);
108 case kWaitingForSampleData
:
109 result
= HaveEnoughDataToEnqueueSamples();
111 ChangeState(kEmittingSamples
);
114 case kEmittingSamples
:
115 result
= EnqueueSample(&audio_buffers
, &video_buffers
, &err
);
117 int64 max_clear
= runs_
->GetMaxClearOffset() + moof_head_
;
118 err
= !ReadAndDiscardMDATsUntil(max_clear
);
122 } while (result
&& !err
);
125 err
= !SendAndFlushSamples(&audio_buffers
, &video_buffers
);
128 DLOG(ERROR
) << "Error while parsing MP4";
138 bool MP4StreamParser::ParseBox(bool* err
) {
141 queue_
.Peek(&buf
, &size
);
142 if (!size
) return false;
144 scoped_ptr
<BoxReader
> reader(
145 BoxReader::ReadTopLevelBox(buf
, size
, log_cb_
, err
));
146 if (reader
.get() == NULL
) return false;
148 if (reader
->type() == FOURCC_MOOV
) {
149 *err
= !ParseMoov(reader
.get());
150 } else if (reader
->type() == FOURCC_MOOF
) {
151 moof_head_
= queue_
.head();
152 *err
= !ParseMoof(reader
.get());
154 // Set up first mdat offset for ReadMDATsUntil().
155 mdat_tail_
= queue_
.head() + reader
->size();
157 // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
158 // be located anywhere in the file, including inside the 'moof' itself.
159 // (Since 'default-base-is-moof' is mandated, no data references can come
160 // before the head of the 'moof', so keeping this box around is sufficient.)
163 MEDIA_LOG(DEBUG
, log_cb_
) << "Skipping unrecognized top-level box: "
164 << FourCCToString(reader
->type());
167 queue_
.Pop(reader
->size());
172 bool MP4StreamParser::ParseMoov(BoxReader
* reader
) {
173 moov_
.reset(new Movie
);
174 RCHECK(moov_
->Parse(reader
));
180 AudioDecoderConfig audio_config
;
181 VideoDecoderConfig video_config
;
183 for (std::vector
<Track
>::const_iterator track
= moov_
->tracks
.begin();
184 track
!= moov_
->tracks
.end(); ++track
) {
185 // TODO(strobe): Only the first audio and video track present in a file are
186 // used. (Track selection is better accomplished via Source IDs, though, so
187 // adding support for track selection within a stream is low-priority.)
188 const SampleDescription
& samp_descr
=
189 track
->media
.information
.sample_table
.description
;
191 // TODO(strobe): When codec reconfigurations are supported, detect and send
192 // a codec reconfiguration for fragments using a sample description index
193 // different from the previous one
195 for (size_t t
= 0; t
< moov_
->extends
.tracks
.size(); t
++) {
196 const TrackExtends
& trex
= moov_
->extends
.tracks
[t
];
197 if (trex
.track_id
== track
->header
.track_id
) {
198 desc_idx
= trex
.default_sample_description_index
;
202 RCHECK(desc_idx
> 0);
203 desc_idx
-= 1; // BMFF descriptor index is one-based
205 if (track
->media
.handler
.type
== kAudio
&& !audio_config
.IsValidConfig()) {
206 RCHECK(!samp_descr
.audio_entries
.empty());
208 // It is not uncommon to find otherwise-valid files with incorrect sample
209 // description indices, so we fail gracefully in that case.
210 if (desc_idx
>= samp_descr
.audio_entries
.size())
212 const AudioSampleEntry
& entry
= samp_descr
.audio_entries
[desc_idx
];
213 const AAC
& aac
= entry
.esds
.aac
;
215 if (!(entry
.format
== FOURCC_MP4A
||
216 (entry
.format
== FOURCC_ENCA
&&
217 entry
.sinf
.format
.format
== FOURCC_MP4A
))) {
218 MEDIA_LOG(ERROR
, log_cb_
) << "Unsupported audio format 0x" << std::hex
219 << entry
.format
<< " in stsd box.";
223 uint8 audio_type
= entry
.esds
.object_type
;
224 DVLOG(1) << "audio_type " << std::hex
<< static_cast<int>(audio_type
);
225 if (audio_object_types_
.find(audio_type
) == audio_object_types_
.end()) {
226 MEDIA_LOG(ERROR
, log_cb_
) << "audio object type 0x" << std::hex
228 << " does not match what is specified in the"
233 AudioCodec codec
= kUnknownAudioCodec
;
234 ChannelLayout channel_layout
= CHANNEL_LAYOUT_NONE
;
235 int sample_per_second
= 0;
236 std::vector
<uint8
> extra_data
;
237 // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
238 // supported MPEG2 AAC varients.
239 if (ESDescriptor::IsAAC(audio_type
)) {
241 channel_layout
= aac
.GetChannelLayout(has_sbr_
);
242 sample_per_second
= aac
.GetOutputSamplesPerSecond(has_sbr_
);
243 #if defined(OS_ANDROID)
244 extra_data
= aac
.codec_specific_data();
247 MEDIA_LOG(ERROR
, log_cb_
) << "Unsupported audio object type 0x"
248 << std::hex
<< audio_type
<< " in esds.";
252 SampleFormat sample_format
;
253 if (entry
.samplesize
== 8) {
254 sample_format
= kSampleFormatU8
;
255 } else if (entry
.samplesize
== 16) {
256 sample_format
= kSampleFormatS16
;
257 } else if (entry
.samplesize
== 32) {
258 sample_format
= kSampleFormatS32
;
260 LOG(ERROR
) << "Unsupported sample size.";
264 is_audio_track_encrypted_
= entry
.sinf
.info
.track_encryption
.is_encrypted
;
265 DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_
;
266 audio_config
.Initialize(
267 codec
, sample_format
, channel_layout
, sample_per_second
,
268 extra_data
.size() ? &extra_data
[0] : NULL
, extra_data
.size(),
269 is_audio_track_encrypted_
, false, base::TimeDelta(),
272 audio_track_id_
= track
->header
.track_id
;
274 if (track
->media
.handler
.type
== kVideo
&& !video_config
.IsValidConfig()) {
275 RCHECK(!samp_descr
.video_entries
.empty());
276 if (desc_idx
>= samp_descr
.video_entries
.size())
278 const VideoSampleEntry
& entry
= samp_descr
.video_entries
[desc_idx
];
280 if (!entry
.IsFormatValid()) {
281 MEDIA_LOG(ERROR
, log_cb_
) << "Unsupported video format 0x" << std::hex
282 << entry
.format
<< " in stsd box.";
286 // TODO(strobe): Recover correct crop box
287 gfx::Size
coded_size(entry
.width
, entry
.height
);
288 gfx::Rect
visible_rect(coded_size
);
289 gfx::Size natural_size
= GetNaturalSize(visible_rect
.size(),
290 entry
.pixel_aspect
.h_spacing
,
291 entry
.pixel_aspect
.v_spacing
);
292 is_video_track_encrypted_
= entry
.sinf
.info
.track_encryption
.is_encrypted
;
293 DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_
;
294 video_config
.Initialize(kCodecH264
, H264PROFILE_MAIN
, VideoFrame::YV12
,
295 coded_size
, visible_rect
, natural_size
,
296 // No decoder-specific buffer needed for AVC;
297 // SPS/PPS are embedded in the video stream
298 NULL
, 0, is_video_track_encrypted_
, false);
300 video_track_id_
= track
->header
.track_id
;
304 RCHECK(config_cb_
.Run(audio_config
, video_config
, TextTrackConfigMap()));
306 StreamParser::InitParameters
params(kInfiniteDuration());
307 if (moov_
->extends
.header
.fragment_duration
> 0) {
308 params
.duration
= TimeDeltaFromRational(
309 moov_
->extends
.header
.fragment_duration
, moov_
->header
.timescale
);
310 } else if (moov_
->header
.duration
> 0 &&
311 moov_
->header
.duration
!= kuint64max
) {
313 TimeDeltaFromRational(moov_
->header
.duration
, moov_
->header
.timescale
);
316 if (!init_cb_
.is_null())
317 base::ResetAndReturn(&init_cb_
).Run(params
);
319 if (!moov_
->pssh
.empty())
320 OnEncryptedMediaInitData(moov_
->pssh
);
325 bool MP4StreamParser::ParseMoof(BoxReader
* reader
) {
326 RCHECK(moov_
.get()); // Must already have initialization segment
328 RCHECK(moof
.Parse(reader
));
330 runs_
.reset(new TrackRunIterator(moov_
.get(), log_cb_
));
331 RCHECK(runs_
->Init(moof
));
332 RCHECK(ComputeHighestEndOffset(moof
));
334 if (!moof
.pssh
.empty())
335 OnEncryptedMediaInitData(moof
.pssh
);
337 new_segment_cb_
.Run();
338 ChangeState(kWaitingForSampleData
);
342 void MP4StreamParser::OnEncryptedMediaInitData(
343 const std::vector
<ProtectionSystemSpecificHeader
>& headers
) {
344 // TODO(strobe): ensure that the value of init_data (all PSSH headers
345 // concatenated in arbitrary order) matches the EME spec.
346 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
347 size_t total_size
= 0;
348 for (size_t i
= 0; i
< headers
.size(); i
++)
349 total_size
+= headers
[i
].raw_box
.size();
351 std::vector
<uint8
> init_data(total_size
);
353 for (size_t i
= 0; i
< headers
.size(); i
++) {
354 memcpy(&init_data
[pos
], &headers
[i
].raw_box
[0],
355 headers
[i
].raw_box
.size());
356 pos
+= headers
[i
].raw_box
.size();
358 encrypted_media_init_data_cb_
.Run(EmeInitDataType::CENC
, init_data
);
361 bool MP4StreamParser::PrepareAVCBuffer(
362 const AVCDecoderConfigurationRecord
& avc_config
,
363 std::vector
<uint8
>* frame_buf
,
364 std::vector
<SubsampleEntry
>* subsamples
) const {
365 // Convert the AVC NALU length fields to Annex B headers, as expected by
366 // decoding libraries. Since this may enlarge the size of the buffer, we also
367 // update the clear byte count for each subsample if encryption is used to
368 // account for the difference in size between the length prefix and Annex B
370 RCHECK(AVC::ConvertFrameToAnnexB(avc_config
.length_size
, frame_buf
));
371 if (!subsamples
->empty()) {
372 const int nalu_size_diff
= 4 - avc_config
.length_size
;
373 size_t expected_size
= runs_
->sample_size() +
374 subsamples
->size() * nalu_size_diff
;
375 RCHECK(frame_buf
->size() == expected_size
);
376 for (size_t i
= 0; i
< subsamples
->size(); i
++)
377 (*subsamples
)[i
].clear_bytes
+= nalu_size_diff
;
380 if (runs_
->is_keyframe()) {
381 // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
382 // a frame. If subsample info is present, we also update the clear byte
383 // count for that first subsample.
384 RCHECK(AVC::InsertParamSetsAnnexB(avc_config
, frame_buf
, subsamples
));
387 DCHECK(AVC::IsValidAnnexB(*frame_buf
, *subsamples
));
391 bool MP4StreamParser::PrepareAACBuffer(
392 const AAC
& aac_config
, std::vector
<uint8
>* frame_buf
,
393 std::vector
<SubsampleEntry
>* subsamples
) const {
394 // Append an ADTS header to every audio sample.
395 RCHECK(aac_config
.ConvertEsdsToADTS(frame_buf
));
397 // As above, adjust subsample information to account for the headers. AAC is
398 // not required to use subsample encryption, so we may need to add an entry.
399 if (subsamples
->empty()) {
400 subsamples
->push_back(SubsampleEntry(
401 kADTSHeaderMinSize
, frame_buf
->size() - kADTSHeaderMinSize
));
403 (*subsamples
)[0].clear_bytes
+= kADTSHeaderMinSize
;
408 bool MP4StreamParser::EnqueueSample(BufferQueue
* audio_buffers
,
409 BufferQueue
* video_buffers
,
411 DCHECK_EQ(state_
, kEmittingSamples
);
413 if (!runs_
->IsRunValid()) {
414 // Flush any buffers we've gotten in this chunk so that buffers don't
415 // cross NewSegment() calls
416 *err
= !SendAndFlushSamples(audio_buffers
, video_buffers
);
420 // Remain in kEmittingSamples state, discarding data, until the end of
421 // the current 'mdat' box has been appended to the queue.
422 if (!queue_
.Trim(mdat_tail_
))
425 ChangeState(kParsingBoxes
);
426 end_of_segment_cb_
.Run();
430 if (!runs_
->IsSampleValid()) {
439 queue_
.Peek(&buf
, &buf_size
);
440 if (!buf_size
) return false;
442 bool audio
= has_audio_
&& audio_track_id_
== runs_
->track_id();
443 bool video
= has_video_
&& video_track_id_
== runs_
->track_id();
445 // Skip this entire track if it's not one we're interested in
446 if (!audio
&& !video
) {
451 // Attempt to cache the auxiliary information first. Aux info is usually
452 // placed in a contiguous block before the sample data, rather than being
453 // interleaved. If we didn't cache it, this would require that we retain the
454 // start of the segment buffer while reading samples. Aux info is typically
455 // quite small compared to sample data, so this pattern is useful on
456 // memory-constrained devices where the source buffer consumes a substantial
457 // portion of the total system memory.
458 if (runs_
->AuxInfoNeedsToBeCached()) {
459 queue_
.PeekAt(runs_
->aux_info_offset() + moof_head_
, &buf
, &buf_size
);
460 if (buf_size
< runs_
->aux_info_size()) return false;
461 *err
= !runs_
->CacheAuxInfo(buf
, buf_size
);
465 queue_
.PeekAt(runs_
->sample_offset() + moof_head_
, &buf
, &buf_size
);
466 if (buf_size
< runs_
->sample_size()) return false;
468 scoped_ptr
<DecryptConfig
> decrypt_config
;
469 std::vector
<SubsampleEntry
> subsamples
;
470 if (runs_
->is_encrypted()) {
471 decrypt_config
= runs_
->GetDecryptConfig();
472 if (!decrypt_config
) {
476 subsamples
= decrypt_config
->subsamples();
479 std::vector
<uint8
> frame_buf(buf
, buf
+ runs_
->sample_size());
481 if (!PrepareAVCBuffer(runs_
->video_description().avcc
,
482 &frame_buf
, &subsamples
)) {
483 MEDIA_LOG(ERROR
, log_cb_
) << "Failed to prepare AVC sample for decode";
490 if (ESDescriptor::IsAAC(runs_
->audio_description().esds
.object_type
) &&
491 !PrepareAACBuffer(runs_
->audio_description().esds
.aac
,
492 &frame_buf
, &subsamples
)) {
493 MEDIA_LOG(ERROR
, log_cb_
) << "Failed to prepare AAC sample for decode";
499 if (decrypt_config
) {
500 if (!subsamples
.empty()) {
501 // Create a new config with the updated subsamples.
502 decrypt_config
.reset(new DecryptConfig(
503 decrypt_config
->key_id(),
504 decrypt_config
->iv(),
507 // else, use the existing config.
508 } else if ((audio
&& is_audio_track_encrypted_
) ||
509 (video
&& is_video_track_encrypted_
)) {
510 // The media pipeline requires a DecryptConfig with an empty |iv|.
511 // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
512 decrypt_config
.reset(
513 new DecryptConfig("1", "", std::vector
<SubsampleEntry
>()));
516 StreamParserBuffer::Type buffer_type
= audio
? DemuxerStream::AUDIO
:
517 DemuxerStream::VIDEO
;
519 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId
520 // type and allow multiple tracks for same media type, if applicable. See
521 // https://crbug.com/341581.
523 // NOTE: MPEG's "random access point" concept is equivalent to the
524 // downstream code's "is keyframe" concept.
525 scoped_refptr
<StreamParserBuffer
> stream_buf
=
526 StreamParserBuffer::CopyFrom(&frame_buf
[0], frame_buf
.size(),
527 runs_
->is_random_access_point(),
531 stream_buf
->set_decrypt_config(decrypt_config
.Pass());
533 stream_buf
->set_duration(runs_
->duration());
534 stream_buf
->set_timestamp(runs_
->cts());
535 stream_buf
->SetDecodeTimestamp(runs_
->dts());
537 DVLOG(3) << "Pushing frame: aud=" << audio
538 << ", key=" << runs_
->is_keyframe()
539 << ", rap=" << runs_
->is_random_access_point()
540 << ", dur=" << runs_
->duration().InMilliseconds()
541 << ", dts=" << runs_
->dts().InMilliseconds()
542 << ", cts=" << runs_
->cts().InMilliseconds()
543 << ", size=" << runs_
->sample_size();
546 audio_buffers
->push_back(stream_buf
);
548 video_buffers
->push_back(stream_buf
);
551 runs_
->AdvanceSample();
555 bool MP4StreamParser::SendAndFlushSamples(BufferQueue
* audio_buffers
,
556 BufferQueue
* video_buffers
) {
557 if (audio_buffers
->empty() && video_buffers
->empty())
560 TextBufferQueueMap empty_text_map
;
561 bool success
= new_buffers_cb_
.Run(*audio_buffers
,
564 audio_buffers
->clear();
565 video_buffers
->clear();
569 bool MP4StreamParser::ReadAndDiscardMDATsUntil(int64 max_clear_offset
) {
571 int64 upper_bound
= std::min(max_clear_offset
, queue_
.tail());
572 while (mdat_tail_
< upper_bound
) {
573 const uint8
* buf
= NULL
;
575 queue_
.PeekAt(mdat_tail_
, &buf
, &size
);
579 if (!BoxReader::StartTopLevelBox(buf
, size
, log_cb_
,
580 &type
, &box_sz
, &err
))
583 if (type
!= FOURCC_MDAT
) {
584 MEDIA_LOG(DEBUG
, log_cb_
) << "Unexpected box type while parsing MDATs: "
585 << FourCCToString(type
);
587 mdat_tail_
+= box_sz
;
589 queue_
.Trim(std::min(mdat_tail_
, upper_bound
));
593 void MP4StreamParser::ChangeState(State new_state
) {
594 DVLOG(2) << "Changing state: " << new_state
;
598 bool MP4StreamParser::HaveEnoughDataToEnqueueSamples() {
599 DCHECK_EQ(state_
, kWaitingForSampleData
);
600 // For muxed content, make sure we have data up to |highest_end_offset_|
601 // so we can ensure proper enqueuing behavior. Otherwise assume we have enough
602 // data and allow per sample offset checks to meter sample enqueuing.
603 // TODO(acolwell): Fix trun box handling so we don't have to special case
605 return !(has_audio_
&& has_video_
&&
606 queue_
.tail() < highest_end_offset_
+ moof_head_
);
609 bool MP4StreamParser::ComputeHighestEndOffset(const MovieFragment
& moof
) {
610 highest_end_offset_
= 0;
612 TrackRunIterator
runs(moov_
.get(), log_cb_
);
613 RCHECK(runs
.Init(moof
));
615 while (runs
.IsRunValid()) {
616 int64 aux_info_end_offset
= runs
.aux_info_offset() + runs
.aux_info_size();
617 if (aux_info_end_offset
> highest_end_offset_
)
618 highest_end_offset_
= aux_info_end_offset
;
620 while (runs
.IsSampleValid()) {
621 int64 sample_end_offset
= runs
.sample_offset() + runs
.sample_size();
622 if (sample_end_offset
> highest_end_offset_
)
623 highest_end_offset_
= sample_end_offset
;
625 runs
.AdvanceSample();