1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/formats/mp4/mp4_stream_parser.h"
7 #include "base/callback.h"
8 #include "base/callback_helpers.h"
9 #include "base/logging.h"
10 #include "base/time/time.h"
11 #include "media/base/audio_decoder_config.h"
12 #include "media/base/stream_parser_buffer.h"
13 #include "media/base/text_track_config.h"
14 #include "media/base/video_decoder_config.h"
15 #include "media/base/video_util.h"
16 #include "media/formats/mp4/box_definitions.h"
17 #include "media/formats/mp4/box_reader.h"
18 #include "media/formats/mp4/es_descriptor.h"
19 #include "media/formats/mp4/rcheck.h"
20 #include "media/formats/mpeg/adts_constants.h"
25 static const char kCencInitDataType
[] = "cenc";
27 MP4StreamParser::MP4StreamParser(const std::set
<int>& audio_object_types
,
29 : state_(kWaitingForInit
),
32 highest_end_offset_(0),
37 audio_object_types_(audio_object_types
),
39 is_audio_track_encrypted_(false),
40 is_video_track_encrypted_(false) {
43 MP4StreamParser::~MP4StreamParser() {}
45 void MP4StreamParser::Init(
46 const InitCB
& init_cb
,
47 const NewConfigCB
& config_cb
,
48 const NewBuffersCB
& new_buffers_cb
,
49 bool /* ignore_text_tracks */,
50 const EncryptedMediaInitDataCB
& encrypted_media_init_data_cb
,
51 const NewMediaSegmentCB
& new_segment_cb
,
52 const base::Closure
& end_of_segment_cb
,
53 const LogCB
& log_cb
) {
54 DCHECK_EQ(state_
, kWaitingForInit
);
55 DCHECK(init_cb_
.is_null());
56 DCHECK(!init_cb
.is_null());
57 DCHECK(!config_cb
.is_null());
58 DCHECK(!new_buffers_cb
.is_null());
59 DCHECK(!encrypted_media_init_data_cb
.is_null());
60 DCHECK(!end_of_segment_cb
.is_null());
62 ChangeState(kParsingBoxes
);
64 config_cb_
= config_cb
;
65 new_buffers_cb_
= new_buffers_cb
;
66 encrypted_media_init_data_cb_
= encrypted_media_init_data_cb
;
67 new_segment_cb_
= new_segment_cb
;
68 end_of_segment_cb_
= end_of_segment_cb
;
72 void MP4StreamParser::Reset() {
79 void MP4StreamParser::Flush() {
80 DCHECK_NE(state_
, kWaitingForInit
);
82 ChangeState(kParsingBoxes
);
85 bool MP4StreamParser::Parse(const uint8
* buf
, int size
) {
86 DCHECK_NE(state_
, kWaitingForInit
);
91 queue_
.Push(buf
, size
);
93 BufferQueue audio_buffers
;
94 BufferQueue video_buffers
;
101 case kWaitingForInit
:
107 result
= ParseBox(&err
);
110 case kWaitingForSampleData
:
111 result
= HaveEnoughDataToEnqueueSamples();
113 ChangeState(kEmittingSamples
);
116 case kEmittingSamples
:
117 result
= EnqueueSample(&audio_buffers
, &video_buffers
, &err
);
119 int64 max_clear
= runs_
->GetMaxClearOffset() + moof_head_
;
120 err
= !ReadAndDiscardMDATsUntil(max_clear
);
124 } while (result
&& !err
);
127 err
= !SendAndFlushSamples(&audio_buffers
, &video_buffers
);
130 DLOG(ERROR
) << "Error while parsing MP4";
140 bool MP4StreamParser::ParseBox(bool* err
) {
143 queue_
.Peek(&buf
, &size
);
144 if (!size
) return false;
146 scoped_ptr
<BoxReader
> reader(
147 BoxReader::ReadTopLevelBox(buf
, size
, log_cb_
, err
));
148 if (reader
.get() == NULL
) return false;
150 if (reader
->type() == FOURCC_MOOV
) {
151 *err
= !ParseMoov(reader
.get());
152 } else if (reader
->type() == FOURCC_MOOF
) {
153 moof_head_
= queue_
.head();
154 *err
= !ParseMoof(reader
.get());
156 // Set up first mdat offset for ReadMDATsUntil().
157 mdat_tail_
= queue_
.head() + reader
->size();
159 // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
160 // be located anywhere in the file, including inside the 'moof' itself.
161 // (Since 'default-base-is-moof' is mandated, no data references can come
162 // before the head of the 'moof', so keeping this box around is sufficient.)
165 MEDIA_LOG(log_cb_
) << "Skipping unrecognized top-level box: "
166 << FourCCToString(reader
->type());
169 queue_
.Pop(reader
->size());
174 bool MP4StreamParser::ParseMoov(BoxReader
* reader
) {
175 moov_
.reset(new Movie
);
176 RCHECK(moov_
->Parse(reader
));
182 AudioDecoderConfig audio_config
;
183 VideoDecoderConfig video_config
;
185 for (std::vector
<Track
>::const_iterator track
= moov_
->tracks
.begin();
186 track
!= moov_
->tracks
.end(); ++track
) {
187 // TODO(strobe): Only the first audio and video track present in a file are
188 // used. (Track selection is better accomplished via Source IDs, though, so
189 // adding support for track selection within a stream is low-priority.)
190 const SampleDescription
& samp_descr
=
191 track
->media
.information
.sample_table
.description
;
193 // TODO(strobe): When codec reconfigurations are supported, detect and send
194 // a codec reconfiguration for fragments using a sample description index
195 // different from the previous one
197 for (size_t t
= 0; t
< moov_
->extends
.tracks
.size(); t
++) {
198 const TrackExtends
& trex
= moov_
->extends
.tracks
[t
];
199 if (trex
.track_id
== track
->header
.track_id
) {
200 desc_idx
= trex
.default_sample_description_index
;
204 RCHECK(desc_idx
> 0);
205 desc_idx
-= 1; // BMFF descriptor index is one-based
207 if (track
->media
.handler
.type
== kAudio
&& !audio_config
.IsValidConfig()) {
208 RCHECK(!samp_descr
.audio_entries
.empty());
210 // It is not uncommon to find otherwise-valid files with incorrect sample
211 // description indices, so we fail gracefully in that case.
212 if (desc_idx
>= samp_descr
.audio_entries
.size())
214 const AudioSampleEntry
& entry
= samp_descr
.audio_entries
[desc_idx
];
215 const AAC
& aac
= entry
.esds
.aac
;
217 if (!(entry
.format
== FOURCC_MP4A
||
218 (entry
.format
== FOURCC_ENCA
&&
219 entry
.sinf
.format
.format
== FOURCC_MP4A
))) {
220 MEDIA_LOG(log_cb_
) << "Unsupported audio format 0x"
221 << std::hex
<< entry
.format
<< " in stsd box.";
225 uint8 audio_type
= entry
.esds
.object_type
;
226 DVLOG(1) << "audio_type " << std::hex
<< static_cast<int>(audio_type
);
227 if (audio_object_types_
.find(audio_type
) == audio_object_types_
.end()) {
228 MEDIA_LOG(log_cb_
) << "audio object type 0x" << std::hex
<< audio_type
229 << " does not match what is specified in the"
234 AudioCodec codec
= kUnknownAudioCodec
;
235 ChannelLayout channel_layout
= CHANNEL_LAYOUT_NONE
;
236 int sample_per_second
= 0;
237 std::vector
<uint8
> extra_data
;
238 // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
239 // supported MPEG2 AAC varients.
240 if (ESDescriptor::IsAAC(audio_type
)) {
242 channel_layout
= aac
.GetChannelLayout(has_sbr_
);
243 sample_per_second
= aac
.GetOutputSamplesPerSecond(has_sbr_
);
244 #if defined(OS_ANDROID)
245 extra_data
= aac
.codec_specific_data();
248 MEDIA_LOG(log_cb_
) << "Unsupported audio object type 0x" << std::hex
249 << audio_type
<< " in esds.";
253 SampleFormat sample_format
;
254 if (entry
.samplesize
== 8) {
255 sample_format
= kSampleFormatU8
;
256 } else if (entry
.samplesize
== 16) {
257 sample_format
= kSampleFormatS16
;
258 } else if (entry
.samplesize
== 32) {
259 sample_format
= kSampleFormatS32
;
261 LOG(ERROR
) << "Unsupported sample size.";
265 is_audio_track_encrypted_
= entry
.sinf
.info
.track_encryption
.is_encrypted
;
266 DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_
;
267 audio_config
.Initialize(
268 codec
, sample_format
, channel_layout
, sample_per_second
,
269 extra_data
.size() ? &extra_data
[0] : NULL
, extra_data
.size(),
270 is_audio_track_encrypted_
, false, base::TimeDelta(),
273 audio_track_id_
= track
->header
.track_id
;
275 if (track
->media
.handler
.type
== kVideo
&& !video_config
.IsValidConfig()) {
276 RCHECK(!samp_descr
.video_entries
.empty());
277 if (desc_idx
>= samp_descr
.video_entries
.size())
279 const VideoSampleEntry
& entry
= samp_descr
.video_entries
[desc_idx
];
281 if (!entry
.IsFormatValid()) {
282 MEDIA_LOG(log_cb_
) << "Unsupported video format 0x"
283 << std::hex
<< entry
.format
<< " in stsd box.";
287 // TODO(strobe): Recover correct crop box
288 gfx::Size
coded_size(entry
.width
, entry
.height
);
289 gfx::Rect
visible_rect(coded_size
);
290 gfx::Size natural_size
= GetNaturalSize(visible_rect
.size(),
291 entry
.pixel_aspect
.h_spacing
,
292 entry
.pixel_aspect
.v_spacing
);
293 is_video_track_encrypted_
= entry
.sinf
.info
.track_encryption
.is_encrypted
;
294 DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_
;
295 video_config
.Initialize(kCodecH264
, H264PROFILE_MAIN
, VideoFrame::YV12
,
296 coded_size
, visible_rect
, natural_size
,
297 // No decoder-specific buffer needed for AVC;
298 // SPS/PPS are embedded in the video stream
299 NULL
, 0, is_video_track_encrypted_
, false);
301 video_track_id_
= track
->header
.track_id
;
305 RCHECK(config_cb_
.Run(audio_config
, video_config
, TextTrackConfigMap()));
307 StreamParser::InitParameters
params(kInfiniteDuration());
308 if (moov_
->extends
.header
.fragment_duration
> 0) {
309 params
.duration
= TimeDeltaFromRational(
310 moov_
->extends
.header
.fragment_duration
, moov_
->header
.timescale
);
311 } else if (moov_
->header
.duration
> 0 &&
312 moov_
->header
.duration
!= kuint64max
) {
314 TimeDeltaFromRational(moov_
->header
.duration
, moov_
->header
.timescale
);
317 if (!init_cb_
.is_null())
318 base::ResetAndReturn(&init_cb_
).Run(true, params
);
320 if (!moov_
->pssh
.empty())
321 OnEncryptedMediaInitData(moov_
->pssh
);
326 bool MP4StreamParser::ParseMoof(BoxReader
* reader
) {
327 RCHECK(moov_
.get()); // Must already have initialization segment
329 RCHECK(moof
.Parse(reader
));
331 runs_
.reset(new TrackRunIterator(moov_
.get(), log_cb_
));
332 RCHECK(runs_
->Init(moof
));
333 RCHECK(ComputeHighestEndOffset(moof
));
335 if (!moof
.pssh
.empty())
336 OnEncryptedMediaInitData(moof
.pssh
);
338 new_segment_cb_
.Run();
339 ChangeState(kWaitingForSampleData
);
343 void MP4StreamParser::OnEncryptedMediaInitData(
344 const std::vector
<ProtectionSystemSpecificHeader
>& headers
) {
345 // TODO(strobe): ensure that the value of init_data (all PSSH headers
346 // concatenated in arbitrary order) matches the EME spec.
347 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
348 size_t total_size
= 0;
349 for (size_t i
= 0; i
< headers
.size(); i
++)
350 total_size
+= headers
[i
].raw_box
.size();
352 std::vector
<uint8
> init_data(total_size
);
354 for (size_t i
= 0; i
< headers
.size(); i
++) {
355 memcpy(&init_data
[pos
], &headers
[i
].raw_box
[0],
356 headers
[i
].raw_box
.size());
357 pos
+= headers
[i
].raw_box
.size();
359 encrypted_media_init_data_cb_
.Run(kCencInitDataType
, init_data
);
362 bool MP4StreamParser::PrepareAVCBuffer(
363 const AVCDecoderConfigurationRecord
& avc_config
,
364 std::vector
<uint8
>* frame_buf
,
365 std::vector
<SubsampleEntry
>* subsamples
) const {
366 // Convert the AVC NALU length fields to Annex B headers, as expected by
367 // decoding libraries. Since this may enlarge the size of the buffer, we also
368 // update the clear byte count for each subsample if encryption is used to
369 // account for the difference in size between the length prefix and Annex B
371 RCHECK(AVC::ConvertFrameToAnnexB(avc_config
.length_size
, frame_buf
));
372 if (!subsamples
->empty()) {
373 const int nalu_size_diff
= 4 - avc_config
.length_size
;
374 size_t expected_size
= runs_
->sample_size() +
375 subsamples
->size() * nalu_size_diff
;
376 RCHECK(frame_buf
->size() == expected_size
);
377 for (size_t i
= 0; i
< subsamples
->size(); i
++)
378 (*subsamples
)[i
].clear_bytes
+= nalu_size_diff
;
381 if (runs_
->is_keyframe()) {
382 // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
383 // a frame. If subsample info is present, we also update the clear byte
384 // count for that first subsample.
385 RCHECK(AVC::InsertParamSetsAnnexB(avc_config
, frame_buf
, subsamples
));
388 DCHECK(AVC::IsValidAnnexB(*frame_buf
, *subsamples
));
392 bool MP4StreamParser::PrepareAACBuffer(
393 const AAC
& aac_config
, std::vector
<uint8
>* frame_buf
,
394 std::vector
<SubsampleEntry
>* subsamples
) const {
395 // Append an ADTS header to every audio sample.
396 RCHECK(aac_config
.ConvertEsdsToADTS(frame_buf
));
398 // As above, adjust subsample information to account for the headers. AAC is
399 // not required to use subsample encryption, so we may need to add an entry.
400 if (subsamples
->empty()) {
401 subsamples
->push_back(SubsampleEntry(
402 kADTSHeaderMinSize
, frame_buf
->size() - kADTSHeaderMinSize
));
404 (*subsamples
)[0].clear_bytes
+= kADTSHeaderMinSize
;
409 bool MP4StreamParser::EnqueueSample(BufferQueue
* audio_buffers
,
410 BufferQueue
* video_buffers
,
412 DCHECK_EQ(state_
, kEmittingSamples
);
414 if (!runs_
->IsRunValid()) {
415 // Flush any buffers we've gotten in this chunk so that buffers don't
416 // cross NewSegment() calls
417 *err
= !SendAndFlushSamples(audio_buffers
, video_buffers
);
421 // Remain in kEmittingSamples state, discarding data, until the end of
422 // the current 'mdat' box has been appended to the queue.
423 if (!queue_
.Trim(mdat_tail_
))
426 ChangeState(kParsingBoxes
);
427 end_of_segment_cb_
.Run();
431 if (!runs_
->IsSampleValid()) {
440 queue_
.Peek(&buf
, &buf_size
);
441 if (!buf_size
) return false;
443 bool audio
= has_audio_
&& audio_track_id_
== runs_
->track_id();
444 bool video
= has_video_
&& video_track_id_
== runs_
->track_id();
446 // Skip this entire track if it's not one we're interested in
447 if (!audio
&& !video
) {
452 // Attempt to cache the auxiliary information first. Aux info is usually
453 // placed in a contiguous block before the sample data, rather than being
454 // interleaved. If we didn't cache it, this would require that we retain the
455 // start of the segment buffer while reading samples. Aux info is typically
456 // quite small compared to sample data, so this pattern is useful on
457 // memory-constrained devices where the source buffer consumes a substantial
458 // portion of the total system memory.
459 if (runs_
->AuxInfoNeedsToBeCached()) {
460 queue_
.PeekAt(runs_
->aux_info_offset() + moof_head_
, &buf
, &buf_size
);
461 if (buf_size
< runs_
->aux_info_size()) return false;
462 *err
= !runs_
->CacheAuxInfo(buf
, buf_size
);
466 queue_
.PeekAt(runs_
->sample_offset() + moof_head_
, &buf
, &buf_size
);
467 if (buf_size
< runs_
->sample_size()) return false;
469 scoped_ptr
<DecryptConfig
> decrypt_config
;
470 std::vector
<SubsampleEntry
> subsamples
;
471 if (runs_
->is_encrypted()) {
472 decrypt_config
= runs_
->GetDecryptConfig();
473 if (!decrypt_config
) {
477 subsamples
= decrypt_config
->subsamples();
480 std::vector
<uint8
> frame_buf(buf
, buf
+ runs_
->sample_size());
482 if (!PrepareAVCBuffer(runs_
->video_description().avcc
,
483 &frame_buf
, &subsamples
)) {
484 MEDIA_LOG(log_cb_
) << "Failed to prepare AVC sample for decode";
491 if (ESDescriptor::IsAAC(runs_
->audio_description().esds
.object_type
) &&
492 !PrepareAACBuffer(runs_
->audio_description().esds
.aac
,
493 &frame_buf
, &subsamples
)) {
494 MEDIA_LOG(log_cb_
) << "Failed to prepare AAC sample for decode";
500 if (decrypt_config
) {
501 if (!subsamples
.empty()) {
502 // Create a new config with the updated subsamples.
503 decrypt_config
.reset(new DecryptConfig(
504 decrypt_config
->key_id(),
505 decrypt_config
->iv(),
508 // else, use the existing config.
509 } else if ((audio
&& is_audio_track_encrypted_
) ||
510 (video
&& is_video_track_encrypted_
)) {
511 // The media pipeline requires a DecryptConfig with an empty |iv|.
512 // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
513 decrypt_config
.reset(
514 new DecryptConfig("1", "", std::vector
<SubsampleEntry
>()));
517 StreamParserBuffer::Type buffer_type
= audio
? DemuxerStream::AUDIO
:
518 DemuxerStream::VIDEO
;
520 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId
521 // type and allow multiple tracks for same media type, if applicable. See
522 // https://crbug.com/341581.
524 // NOTE: MPEG's "random access point" concept is equivalent to the
525 // downstream code's "is keyframe" concept.
526 scoped_refptr
<StreamParserBuffer
> stream_buf
=
527 StreamParserBuffer::CopyFrom(&frame_buf
[0], frame_buf
.size(),
528 runs_
->is_random_access_point(),
532 stream_buf
->set_decrypt_config(decrypt_config
.Pass());
534 stream_buf
->set_duration(runs_
->duration());
535 stream_buf
->set_timestamp(runs_
->cts());
536 stream_buf
->SetDecodeTimestamp(runs_
->dts());
538 DVLOG(3) << "Pushing frame: aud=" << audio
539 << ", key=" << runs_
->is_keyframe()
540 << ", rap=" << runs_
->is_random_access_point()
541 << ", dur=" << runs_
->duration().InMilliseconds()
542 << ", dts=" << runs_
->dts().InMilliseconds()
543 << ", cts=" << runs_
->cts().InMilliseconds()
544 << ", size=" << runs_
->sample_size();
547 audio_buffers
->push_back(stream_buf
);
549 video_buffers
->push_back(stream_buf
);
552 runs_
->AdvanceSample();
556 bool MP4StreamParser::SendAndFlushSamples(BufferQueue
* audio_buffers
,
557 BufferQueue
* video_buffers
) {
558 if (audio_buffers
->empty() && video_buffers
->empty())
561 TextBufferQueueMap empty_text_map
;
562 bool success
= new_buffers_cb_
.Run(*audio_buffers
,
565 audio_buffers
->clear();
566 video_buffers
->clear();
570 bool MP4StreamParser::ReadAndDiscardMDATsUntil(int64 max_clear_offset
) {
572 int64 upper_bound
= std::min(max_clear_offset
, queue_
.tail());
573 while (mdat_tail_
< upper_bound
) {
574 const uint8
* buf
= NULL
;
576 queue_
.PeekAt(mdat_tail_
, &buf
, &size
);
580 if (!BoxReader::StartTopLevelBox(buf
, size
, log_cb_
,
581 &type
, &box_sz
, &err
))
584 if (type
!= FOURCC_MDAT
) {
585 MEDIA_LOG(log_cb_
) << "Unexpected box type while parsing MDATs: "
586 << FourCCToString(type
);
588 mdat_tail_
+= box_sz
;
590 queue_
.Trim(std::min(mdat_tail_
, upper_bound
));
594 void MP4StreamParser::ChangeState(State new_state
) {
595 DVLOG(2) << "Changing state: " << new_state
;
599 bool MP4StreamParser::HaveEnoughDataToEnqueueSamples() {
600 DCHECK_EQ(state_
, kWaitingForSampleData
);
601 // For muxed content, make sure we have data up to |highest_end_offset_|
602 // so we can ensure proper enqueuing behavior. Otherwise assume we have enough
603 // data and allow per sample offset checks to meter sample enqueuing.
604 // TODO(acolwell): Fix trun box handling so we don't have to special case
606 return !(has_audio_
&& has_video_
&&
607 queue_
.tail() < highest_end_offset_
+ moof_head_
);
610 bool MP4StreamParser::ComputeHighestEndOffset(const MovieFragment
& moof
) {
611 highest_end_offset_
= 0;
613 TrackRunIterator
runs(moov_
.get(), log_cb_
);
614 RCHECK(runs
.Init(moof
));
616 while (runs
.IsRunValid()) {
617 int64 aux_info_end_offset
= runs
.aux_info_offset() + runs
.aux_info_size();
618 if (aux_info_end_offset
> highest_end_offset_
)
619 highest_end_offset_
= aux_info_end_offset
;
621 while (runs
.IsSampleValid()) {
622 int64 sample_end_offset
= runs
.sample_offset() + runs
.sample_size();
623 if (sample_end_offset
> highest_end_offset_
)
624 highest_end_offset_
= sample_end_offset
;
626 runs
.AdvanceSample();