Delete chrome.mediaGalleriesPrivate because the functionality unique to it has since...
[chromium-blink-merge.git] / media / formats / mp4 / mp4_stream_parser.cc
blobd4f63cbd3b76b4d01c8144e564a7bee685791409
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/formats/mp4/mp4_stream_parser.h"
7 #include "base/callback.h"
8 #include "base/callback_helpers.h"
9 #include "base/logging.h"
10 #include "base/time/time.h"
11 #include "media/base/audio_decoder_config.h"
12 #include "media/base/stream_parser_buffer.h"
13 #include "media/base/text_track_config.h"
14 #include "media/base/video_decoder_config.h"
15 #include "media/base/video_util.h"
16 #include "media/formats/mp4/box_definitions.h"
17 #include "media/formats/mp4/box_reader.h"
18 #include "media/formats/mp4/es_descriptor.h"
19 #include "media/formats/mp4/rcheck.h"
20 #include "media/formats/mpeg/adts_constants.h"
22 namespace media {
23 namespace mp4 {
25 static const char kCencInitDataType[] = "cenc";
27 MP4StreamParser::MP4StreamParser(const std::set<int>& audio_object_types,
28 bool has_sbr)
29 : state_(kWaitingForInit),
30 moof_head_(0),
31 mdat_tail_(0),
32 highest_end_offset_(0),
33 has_audio_(false),
34 has_video_(false),
35 audio_track_id_(0),
36 video_track_id_(0),
37 audio_object_types_(audio_object_types),
38 has_sbr_(has_sbr),
39 is_audio_track_encrypted_(false),
40 is_video_track_encrypted_(false) {
43 MP4StreamParser::~MP4StreamParser() {}
45 void MP4StreamParser::Init(
46 const InitCB& init_cb,
47 const NewConfigCB& config_cb,
48 const NewBuffersCB& new_buffers_cb,
49 bool /* ignore_text_tracks */,
50 const EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
51 const NewMediaSegmentCB& new_segment_cb,
52 const base::Closure& end_of_segment_cb,
53 const LogCB& log_cb) {
54 DCHECK_EQ(state_, kWaitingForInit);
55 DCHECK(init_cb_.is_null());
56 DCHECK(!init_cb.is_null());
57 DCHECK(!config_cb.is_null());
58 DCHECK(!new_buffers_cb.is_null());
59 DCHECK(!encrypted_media_init_data_cb.is_null());
60 DCHECK(!end_of_segment_cb.is_null());
62 ChangeState(kParsingBoxes);
63 init_cb_ = init_cb;
64 config_cb_ = config_cb;
65 new_buffers_cb_ = new_buffers_cb;
66 encrypted_media_init_data_cb_ = encrypted_media_init_data_cb;
67 new_segment_cb_ = new_segment_cb;
68 end_of_segment_cb_ = end_of_segment_cb;
69 log_cb_ = log_cb;
72 void MP4StreamParser::Reset() {
73 queue_.Reset();
74 runs_.reset();
75 moof_head_ = 0;
76 mdat_tail_ = 0;
79 void MP4StreamParser::Flush() {
80 DCHECK_NE(state_, kWaitingForInit);
81 Reset();
82 ChangeState(kParsingBoxes);
85 bool MP4StreamParser::Parse(const uint8* buf, int size) {
86 DCHECK_NE(state_, kWaitingForInit);
88 if (state_ == kError)
89 return false;
91 queue_.Push(buf, size);
93 BufferQueue audio_buffers;
94 BufferQueue video_buffers;
96 bool result = false;
97 bool err = false;
99 do {
100 switch (state_) {
101 case kWaitingForInit:
102 case kError:
103 NOTREACHED();
104 return false;
106 case kParsingBoxes:
107 result = ParseBox(&err);
108 break;
110 case kWaitingForSampleData:
111 result = HaveEnoughDataToEnqueueSamples();
112 if (result)
113 ChangeState(kEmittingSamples);
114 break;
116 case kEmittingSamples:
117 result = EnqueueSample(&audio_buffers, &video_buffers, &err);
118 if (result) {
119 int64 max_clear = runs_->GetMaxClearOffset() + moof_head_;
120 err = !ReadAndDiscardMDATsUntil(max_clear);
122 break;
124 } while (result && !err);
126 if (!err)
127 err = !SendAndFlushSamples(&audio_buffers, &video_buffers);
129 if (err) {
130 DLOG(ERROR) << "Error while parsing MP4";
131 moov_.reset();
132 Reset();
133 ChangeState(kError);
134 return false;
137 return true;
140 bool MP4StreamParser::ParseBox(bool* err) {
141 const uint8* buf;
142 int size;
143 queue_.Peek(&buf, &size);
144 if (!size) return false;
146 scoped_ptr<BoxReader> reader(
147 BoxReader::ReadTopLevelBox(buf, size, log_cb_, err));
148 if (reader.get() == NULL) return false;
150 if (reader->type() == FOURCC_MOOV) {
151 *err = !ParseMoov(reader.get());
152 } else if (reader->type() == FOURCC_MOOF) {
153 moof_head_ = queue_.head();
154 *err = !ParseMoof(reader.get());
156 // Set up first mdat offset for ReadMDATsUntil().
157 mdat_tail_ = queue_.head() + reader->size();
159 // Return early to avoid evicting 'moof' data from queue. Auxiliary info may
160 // be located anywhere in the file, including inside the 'moof' itself.
161 // (Since 'default-base-is-moof' is mandated, no data references can come
162 // before the head of the 'moof', so keeping this box around is sufficient.)
163 return !(*err);
164 } else {
165 MEDIA_LOG(log_cb_) << "Skipping unrecognized top-level box: "
166 << FourCCToString(reader->type());
169 queue_.Pop(reader->size());
170 return !(*err);
174 bool MP4StreamParser::ParseMoov(BoxReader* reader) {
175 moov_.reset(new Movie);
176 RCHECK(moov_->Parse(reader));
177 runs_.reset();
179 has_audio_ = false;
180 has_video_ = false;
182 AudioDecoderConfig audio_config;
183 VideoDecoderConfig video_config;
185 for (std::vector<Track>::const_iterator track = moov_->tracks.begin();
186 track != moov_->tracks.end(); ++track) {
187 // TODO(strobe): Only the first audio and video track present in a file are
188 // used. (Track selection is better accomplished via Source IDs, though, so
189 // adding support for track selection within a stream is low-priority.)
190 const SampleDescription& samp_descr =
191 track->media.information.sample_table.description;
193 // TODO(strobe): When codec reconfigurations are supported, detect and send
194 // a codec reconfiguration for fragments using a sample description index
195 // different from the previous one
196 size_t desc_idx = 0;
197 for (size_t t = 0; t < moov_->extends.tracks.size(); t++) {
198 const TrackExtends& trex = moov_->extends.tracks[t];
199 if (trex.track_id == track->header.track_id) {
200 desc_idx = trex.default_sample_description_index;
201 break;
204 RCHECK(desc_idx > 0);
205 desc_idx -= 1; // BMFF descriptor index is one-based
207 if (track->media.handler.type == kAudio && !audio_config.IsValidConfig()) {
208 RCHECK(!samp_descr.audio_entries.empty());
210 // It is not uncommon to find otherwise-valid files with incorrect sample
211 // description indices, so we fail gracefully in that case.
212 if (desc_idx >= samp_descr.audio_entries.size())
213 desc_idx = 0;
214 const AudioSampleEntry& entry = samp_descr.audio_entries[desc_idx];
215 const AAC& aac = entry.esds.aac;
217 if (!(entry.format == FOURCC_MP4A ||
218 (entry.format == FOURCC_ENCA &&
219 entry.sinf.format.format == FOURCC_MP4A))) {
220 MEDIA_LOG(log_cb_) << "Unsupported audio format 0x"
221 << std::hex << entry.format << " in stsd box.";
222 return false;
225 uint8 audio_type = entry.esds.object_type;
226 DVLOG(1) << "audio_type " << std::hex << static_cast<int>(audio_type);
227 if (audio_object_types_.find(audio_type) == audio_object_types_.end()) {
228 MEDIA_LOG(log_cb_) << "audio object type 0x" << std::hex << audio_type
229 << " does not match what is specified in the"
230 << " mimetype.";
231 return false;
234 AudioCodec codec = kUnknownAudioCodec;
235 ChannelLayout channel_layout = CHANNEL_LAYOUT_NONE;
236 int sample_per_second = 0;
237 std::vector<uint8> extra_data;
238 // Check if it is MPEG4 AAC defined in ISO 14496 Part 3 or
239 // supported MPEG2 AAC varients.
240 if (ESDescriptor::IsAAC(audio_type)) {
241 codec = kCodecAAC;
242 channel_layout = aac.GetChannelLayout(has_sbr_);
243 sample_per_second = aac.GetOutputSamplesPerSecond(has_sbr_);
244 #if defined(OS_ANDROID)
245 extra_data = aac.codec_specific_data();
246 #endif
247 } else {
248 MEDIA_LOG(log_cb_) << "Unsupported audio object type 0x" << std::hex
249 << audio_type << " in esds.";
250 return false;
253 SampleFormat sample_format;
254 if (entry.samplesize == 8) {
255 sample_format = kSampleFormatU8;
256 } else if (entry.samplesize == 16) {
257 sample_format = kSampleFormatS16;
258 } else if (entry.samplesize == 32) {
259 sample_format = kSampleFormatS32;
260 } else {
261 LOG(ERROR) << "Unsupported sample size.";
262 return false;
265 is_audio_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
266 DVLOG(1) << "is_audio_track_encrypted_: " << is_audio_track_encrypted_;
267 audio_config.Initialize(
268 codec, sample_format, channel_layout, sample_per_second,
269 extra_data.size() ? &extra_data[0] : NULL, extra_data.size(),
270 is_audio_track_encrypted_, false, base::TimeDelta(),
272 has_audio_ = true;
273 audio_track_id_ = track->header.track_id;
275 if (track->media.handler.type == kVideo && !video_config.IsValidConfig()) {
276 RCHECK(!samp_descr.video_entries.empty());
277 if (desc_idx >= samp_descr.video_entries.size())
278 desc_idx = 0;
279 const VideoSampleEntry& entry = samp_descr.video_entries[desc_idx];
281 if (!entry.IsFormatValid()) {
282 MEDIA_LOG(log_cb_) << "Unsupported video format 0x"
283 << std::hex << entry.format << " in stsd box.";
284 return false;
287 // TODO(strobe): Recover correct crop box
288 gfx::Size coded_size(entry.width, entry.height);
289 gfx::Rect visible_rect(coded_size);
290 gfx::Size natural_size = GetNaturalSize(visible_rect.size(),
291 entry.pixel_aspect.h_spacing,
292 entry.pixel_aspect.v_spacing);
293 is_video_track_encrypted_ = entry.sinf.info.track_encryption.is_encrypted;
294 DVLOG(1) << "is_video_track_encrypted_: " << is_video_track_encrypted_;
295 video_config.Initialize(kCodecH264, H264PROFILE_MAIN, VideoFrame::YV12,
296 coded_size, visible_rect, natural_size,
297 // No decoder-specific buffer needed for AVC;
298 // SPS/PPS are embedded in the video stream
299 NULL, 0, is_video_track_encrypted_, false);
300 has_video_ = true;
301 video_track_id_ = track->header.track_id;
305 RCHECK(config_cb_.Run(audio_config, video_config, TextTrackConfigMap()));
307 StreamParser::InitParameters params(kInfiniteDuration());
308 if (moov_->extends.header.fragment_duration > 0) {
309 params.duration = TimeDeltaFromRational(
310 moov_->extends.header.fragment_duration, moov_->header.timescale);
311 } else if (moov_->header.duration > 0 &&
312 moov_->header.duration != kuint64max) {
313 params.duration =
314 TimeDeltaFromRational(moov_->header.duration, moov_->header.timescale);
317 if (!init_cb_.is_null())
318 base::ResetAndReturn(&init_cb_).Run(true, params);
320 if (!moov_->pssh.empty())
321 OnEncryptedMediaInitData(moov_->pssh);
323 return true;
326 bool MP4StreamParser::ParseMoof(BoxReader* reader) {
327 RCHECK(moov_.get()); // Must already have initialization segment
328 MovieFragment moof;
329 RCHECK(moof.Parse(reader));
330 if (!runs_)
331 runs_.reset(new TrackRunIterator(moov_.get(), log_cb_));
332 RCHECK(runs_->Init(moof));
333 RCHECK(ComputeHighestEndOffset(moof));
335 if (!moof.pssh.empty())
336 OnEncryptedMediaInitData(moof.pssh);
338 new_segment_cb_.Run();
339 ChangeState(kWaitingForSampleData);
340 return true;
343 void MP4StreamParser::OnEncryptedMediaInitData(
344 const std::vector<ProtectionSystemSpecificHeader>& headers) {
345 // TODO(strobe): ensure that the value of init_data (all PSSH headers
346 // concatenated in arbitrary order) matches the EME spec.
347 // See https://www.w3.org/Bugs/Public/show_bug.cgi?id=17673.
348 size_t total_size = 0;
349 for (size_t i = 0; i < headers.size(); i++)
350 total_size += headers[i].raw_box.size();
352 std::vector<uint8> init_data(total_size);
353 size_t pos = 0;
354 for (size_t i = 0; i < headers.size(); i++) {
355 memcpy(&init_data[pos], &headers[i].raw_box[0],
356 headers[i].raw_box.size());
357 pos += headers[i].raw_box.size();
359 encrypted_media_init_data_cb_.Run(kCencInitDataType, init_data);
362 bool MP4StreamParser::PrepareAVCBuffer(
363 const AVCDecoderConfigurationRecord& avc_config,
364 std::vector<uint8>* frame_buf,
365 std::vector<SubsampleEntry>* subsamples) const {
366 // Convert the AVC NALU length fields to Annex B headers, as expected by
367 // decoding libraries. Since this may enlarge the size of the buffer, we also
368 // update the clear byte count for each subsample if encryption is used to
369 // account for the difference in size between the length prefix and Annex B
370 // start code.
371 RCHECK(AVC::ConvertFrameToAnnexB(avc_config.length_size, frame_buf));
372 if (!subsamples->empty()) {
373 const int nalu_size_diff = 4 - avc_config.length_size;
374 size_t expected_size = runs_->sample_size() +
375 subsamples->size() * nalu_size_diff;
376 RCHECK(frame_buf->size() == expected_size);
377 for (size_t i = 0; i < subsamples->size(); i++)
378 (*subsamples)[i].clear_bytes += nalu_size_diff;
381 if (runs_->is_keyframe()) {
382 // If this is a keyframe, we (re-)inject SPS and PPS headers at the start of
383 // a frame. If subsample info is present, we also update the clear byte
384 // count for that first subsample.
385 RCHECK(AVC::InsertParamSetsAnnexB(avc_config, frame_buf, subsamples));
388 DCHECK(AVC::IsValidAnnexB(*frame_buf, *subsamples));
389 return true;
392 bool MP4StreamParser::PrepareAACBuffer(
393 const AAC& aac_config, std::vector<uint8>* frame_buf,
394 std::vector<SubsampleEntry>* subsamples) const {
395 // Append an ADTS header to every audio sample.
396 RCHECK(aac_config.ConvertEsdsToADTS(frame_buf));
398 // As above, adjust subsample information to account for the headers. AAC is
399 // not required to use subsample encryption, so we may need to add an entry.
400 if (subsamples->empty()) {
401 subsamples->push_back(SubsampleEntry(
402 kADTSHeaderMinSize, frame_buf->size() - kADTSHeaderMinSize));
403 } else {
404 (*subsamples)[0].clear_bytes += kADTSHeaderMinSize;
406 return true;
409 bool MP4StreamParser::EnqueueSample(BufferQueue* audio_buffers,
410 BufferQueue* video_buffers,
411 bool* err) {
412 DCHECK_EQ(state_, kEmittingSamples);
414 if (!runs_->IsRunValid()) {
415 // Flush any buffers we've gotten in this chunk so that buffers don't
416 // cross NewSegment() calls
417 *err = !SendAndFlushSamples(audio_buffers, video_buffers);
418 if (*err)
419 return false;
421 // Remain in kEmittingSamples state, discarding data, until the end of
422 // the current 'mdat' box has been appended to the queue.
423 if (!queue_.Trim(mdat_tail_))
424 return false;
426 ChangeState(kParsingBoxes);
427 end_of_segment_cb_.Run();
428 return true;
431 if (!runs_->IsSampleValid()) {
432 runs_->AdvanceRun();
433 return true;
436 DCHECK(!(*err));
438 const uint8* buf;
439 int buf_size;
440 queue_.Peek(&buf, &buf_size);
441 if (!buf_size) return false;
443 bool audio = has_audio_ && audio_track_id_ == runs_->track_id();
444 bool video = has_video_ && video_track_id_ == runs_->track_id();
446 // Skip this entire track if it's not one we're interested in
447 if (!audio && !video) {
448 runs_->AdvanceRun();
449 return true;
452 // Attempt to cache the auxiliary information first. Aux info is usually
453 // placed in a contiguous block before the sample data, rather than being
454 // interleaved. If we didn't cache it, this would require that we retain the
455 // start of the segment buffer while reading samples. Aux info is typically
456 // quite small compared to sample data, so this pattern is useful on
457 // memory-constrained devices where the source buffer consumes a substantial
458 // portion of the total system memory.
459 if (runs_->AuxInfoNeedsToBeCached()) {
460 queue_.PeekAt(runs_->aux_info_offset() + moof_head_, &buf, &buf_size);
461 if (buf_size < runs_->aux_info_size()) return false;
462 *err = !runs_->CacheAuxInfo(buf, buf_size);
463 return !*err;
466 queue_.PeekAt(runs_->sample_offset() + moof_head_, &buf, &buf_size);
467 if (buf_size < runs_->sample_size()) return false;
469 scoped_ptr<DecryptConfig> decrypt_config;
470 std::vector<SubsampleEntry> subsamples;
471 if (runs_->is_encrypted()) {
472 decrypt_config = runs_->GetDecryptConfig();
473 if (!decrypt_config) {
474 *err = true;
475 return false;
477 subsamples = decrypt_config->subsamples();
480 std::vector<uint8> frame_buf(buf, buf + runs_->sample_size());
481 if (video) {
482 if (!PrepareAVCBuffer(runs_->video_description().avcc,
483 &frame_buf, &subsamples)) {
484 MEDIA_LOG(log_cb_) << "Failed to prepare AVC sample for decode";
485 *err = true;
486 return false;
490 if (audio) {
491 if (ESDescriptor::IsAAC(runs_->audio_description().esds.object_type) &&
492 !PrepareAACBuffer(runs_->audio_description().esds.aac,
493 &frame_buf, &subsamples)) {
494 MEDIA_LOG(log_cb_) << "Failed to prepare AAC sample for decode";
495 *err = true;
496 return false;
500 if (decrypt_config) {
501 if (!subsamples.empty()) {
502 // Create a new config with the updated subsamples.
503 decrypt_config.reset(new DecryptConfig(
504 decrypt_config->key_id(),
505 decrypt_config->iv(),
506 subsamples));
508 // else, use the existing config.
509 } else if ((audio && is_audio_track_encrypted_) ||
510 (video && is_video_track_encrypted_)) {
511 // The media pipeline requires a DecryptConfig with an empty |iv|.
512 // TODO(ddorwin): Refactor so we do not need a fake key ID ("1");
513 decrypt_config.reset(
514 new DecryptConfig("1", "", std::vector<SubsampleEntry>()));
517 StreamParserBuffer::Type buffer_type = audio ? DemuxerStream::AUDIO :
518 DemuxerStream::VIDEO;
520 // TODO(wolenetz/acolwell): Validate and use a common cross-parser TrackId
521 // type and allow multiple tracks for same media type, if applicable. See
522 // https://crbug.com/341581.
524 // NOTE: MPEG's "random access point" concept is equivalent to the
525 // downstream code's "is keyframe" concept.
526 scoped_refptr<StreamParserBuffer> stream_buf =
527 StreamParserBuffer::CopyFrom(&frame_buf[0], frame_buf.size(),
528 runs_->is_random_access_point(),
529 buffer_type, 0);
531 if (decrypt_config)
532 stream_buf->set_decrypt_config(decrypt_config.Pass());
534 stream_buf->set_duration(runs_->duration());
535 stream_buf->set_timestamp(runs_->cts());
536 stream_buf->SetDecodeTimestamp(runs_->dts());
538 DVLOG(3) << "Pushing frame: aud=" << audio
539 << ", key=" << runs_->is_keyframe()
540 << ", rap=" << runs_->is_random_access_point()
541 << ", dur=" << runs_->duration().InMilliseconds()
542 << ", dts=" << runs_->dts().InMilliseconds()
543 << ", cts=" << runs_->cts().InMilliseconds()
544 << ", size=" << runs_->sample_size();
546 if (audio) {
547 audio_buffers->push_back(stream_buf);
548 } else {
549 video_buffers->push_back(stream_buf);
552 runs_->AdvanceSample();
553 return true;
556 bool MP4StreamParser::SendAndFlushSamples(BufferQueue* audio_buffers,
557 BufferQueue* video_buffers) {
558 if (audio_buffers->empty() && video_buffers->empty())
559 return true;
561 TextBufferQueueMap empty_text_map;
562 bool success = new_buffers_cb_.Run(*audio_buffers,
563 *video_buffers,
564 empty_text_map);
565 audio_buffers->clear();
566 video_buffers->clear();
567 return success;
570 bool MP4StreamParser::ReadAndDiscardMDATsUntil(int64 max_clear_offset) {
571 bool err = false;
572 int64 upper_bound = std::min(max_clear_offset, queue_.tail());
573 while (mdat_tail_ < upper_bound) {
574 const uint8* buf = NULL;
575 int size = 0;
576 queue_.PeekAt(mdat_tail_, &buf, &size);
578 FourCC type;
579 int box_sz;
580 if (!BoxReader::StartTopLevelBox(buf, size, log_cb_,
581 &type, &box_sz, &err))
582 break;
584 if (type != FOURCC_MDAT) {
585 MEDIA_LOG(log_cb_) << "Unexpected box type while parsing MDATs: "
586 << FourCCToString(type);
588 mdat_tail_ += box_sz;
590 queue_.Trim(std::min(mdat_tail_, upper_bound));
591 return !err;
594 void MP4StreamParser::ChangeState(State new_state) {
595 DVLOG(2) << "Changing state: " << new_state;
596 state_ = new_state;
599 bool MP4StreamParser::HaveEnoughDataToEnqueueSamples() {
600 DCHECK_EQ(state_, kWaitingForSampleData);
601 // For muxed content, make sure we have data up to |highest_end_offset_|
602 // so we can ensure proper enqueuing behavior. Otherwise assume we have enough
603 // data and allow per sample offset checks to meter sample enqueuing.
604 // TODO(acolwell): Fix trun box handling so we don't have to special case
605 // muxed content.
606 return !(has_audio_ && has_video_ &&
607 queue_.tail() < highest_end_offset_ + moof_head_);
610 bool MP4StreamParser::ComputeHighestEndOffset(const MovieFragment& moof) {
611 highest_end_offset_ = 0;
613 TrackRunIterator runs(moov_.get(), log_cb_);
614 RCHECK(runs.Init(moof));
616 while (runs.IsRunValid()) {
617 int64 aux_info_end_offset = runs.aux_info_offset() + runs.aux_info_size();
618 if (aux_info_end_offset > highest_end_offset_)
619 highest_end_offset_ = aux_info_end_offset;
621 while (runs.IsSampleValid()) {
622 int64 sample_end_offset = runs.sample_offset() + runs.sample_size();
623 if (sample_end_offset > highest_end_offset_)
624 highest_end_offset_ = sample_end_offset;
626 runs.AdvanceSample();
628 runs.AdvanceRun();
631 return true;
634 } // namespace mp4
635 } // namespace media