1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/formats/mp4/track_run_iterator.h"
10 #include "media/formats/mp4/rcheck.h"
11 #include "media/formats/mp4/sample_to_group_iterator.h"
21 uint32 cenc_group_description_index
;
26 std::vector
<SampleInfo
> samples
;
29 int64 sample_start_offset
;
32 const AudioSampleEntry
* audio_description
;
33 const VideoSampleEntry
* video_description
;
34 const SampleGroupDescription
* track_sample_encryption_group
;
36 int64 aux_info_start_offset
; // Only valid if aux_info_total_size > 0.
37 int aux_info_default_size
;
38 std::vector
<uint8
> aux_info_sizes
; // Populated if default_size == 0.
39 int aux_info_total_size
;
41 std::vector
<CencSampleEncryptionInfoEntry
> fragment_sample_encryption_info
;
47 TrackRunInfo::TrackRunInfo()
51 sample_start_offset(-1),
53 aux_info_start_offset(-1),
54 aux_info_default_size(-1),
55 aux_info_total_size(-1) {
57 TrackRunInfo::~TrackRunInfo() {}
59 base::TimeDelta
TimeDeltaFromRational(int64 numer
, int64 denom
) {
60 // To avoid overflow, split the following calculation:
61 // (numer * base::Time::kMicrosecondsPerSecond) / denom
63 // (numer / denom) * base::Time::kMicrosecondsPerSecond +
64 // ((numer % denom) * base::Time::kMicrosecondsPerSecond) / denom
65 int64 a
= numer
/ denom
;
66 DCHECK_LE((a
> 0 ? a
: -a
), kint64max
/ base::Time::kMicrosecondsPerSecond
);
67 int64 timea_in_us
= a
* base::Time::kMicrosecondsPerSecond
;
69 int64 b
= numer
% denom
;
70 DCHECK_LE((b
> 0 ? b
: -b
), kint64max
/ base::Time::kMicrosecondsPerSecond
);
71 int64 timeb_in_us
= (b
* base::Time::kMicrosecondsPerSecond
) / denom
;
73 DCHECK((timeb_in_us
< 0) || (timea_in_us
<= kint64max
- timeb_in_us
));
74 DCHECK((timeb_in_us
> 0) || (timea_in_us
>= kint64min
- timeb_in_us
));
75 return base::TimeDelta::FromMicroseconds(timea_in_us
+ timeb_in_us
);
78 DecodeTimestamp
DecodeTimestampFromRational(int64 numer
, int64 denom
) {
79 return DecodeTimestamp::FromPresentationTime(
80 TimeDeltaFromRational(numer
, denom
));
83 TrackRunIterator::TrackRunIterator(const Movie
* moov
,
84 const scoped_refptr
<MediaLog
>& media_log
)
85 : moov_(moov
), media_log_(media_log
), sample_offset_(0) {
89 TrackRunIterator::~TrackRunIterator() {}
91 static std::string
HexFlags(uint32 flags
) {
92 std::stringstream stream
;
93 stream
<< std::setfill('0') << std::setw(sizeof(flags
)*2) << std::hex
98 static bool PopulateSampleInfo(const TrackExtends
& trex
,
99 const TrackFragmentHeader
& tfhd
,
100 const TrackFragmentRun
& trun
,
101 const int64 edit_list_offset
,
103 SampleInfo
* sample_info
,
104 const SampleDependsOn sdtp_sample_depends_on
,
106 const scoped_refptr
<MediaLog
>& media_log
) {
107 if (i
< trun
.sample_sizes
.size()) {
108 sample_info
->size
= trun
.sample_sizes
[i
];
109 } else if (tfhd
.default_sample_size
> 0) {
110 sample_info
->size
= tfhd
.default_sample_size
;
112 sample_info
->size
= trex
.default_sample_size
;
115 if (i
< trun
.sample_durations
.size()) {
116 sample_info
->duration
= trun
.sample_durations
[i
];
117 } else if (tfhd
.default_sample_duration
> 0) {
118 sample_info
->duration
= tfhd
.default_sample_duration
;
120 sample_info
->duration
= trex
.default_sample_duration
;
123 if (i
< trun
.sample_composition_time_offsets
.size()) {
124 sample_info
->cts_offset
= trun
.sample_composition_time_offsets
[i
];
126 sample_info
->cts_offset
= 0;
128 sample_info
->cts_offset
+= edit_list_offset
;
131 if (i
< trun
.sample_flags
.size()) {
132 flags
= trun
.sample_flags
[i
];
133 DVLOG(4) << __FUNCTION__
<< " trun sample flags " << HexFlags(flags
);
134 } else if (tfhd
.has_default_sample_flags
) {
135 flags
= tfhd
.default_sample_flags
;
136 DVLOG(4) << __FUNCTION__
<< " tfhd sample flags " << HexFlags(flags
);
138 flags
= trex
.default_sample_flags
;
139 DVLOG(4) << __FUNCTION__
<< " trex sample flags " << HexFlags(flags
);
142 SampleDependsOn sample_depends_on
=
143 static_cast<SampleDependsOn
>((flags
>> 24) & 0x3);
144 if (sample_depends_on
== kSampleDependsOnUnknown
) {
145 sample_depends_on
= sdtp_sample_depends_on
;
147 DVLOG(4) << __FUNCTION__
<< " sample_depends_on " << sample_depends_on
;
148 if (sample_depends_on
== kSampleDependsOnReserved
) {
149 MEDIA_LOG(ERROR
, media_log
) << "Reserved value used in sample dependency"
154 // Per spec (ISO 14496-12:2012), the definition for a "sync sample" is
155 // equivalent to the downstream code's "is keyframe" concept. But media exists
156 // that marks non-key video frames as sync samples (http://crbug.com/507916
157 // and http://crbug.com/310712). Hence, for video we additionally check that
158 // the sample does not depend on others (FFmpeg does too, see mov_read_trun).
159 // Sample dependency is not ignored for audio because encoded audio samples
160 // can depend on other samples and still be used for random access. Generally
161 // all audio samples are expected to be sync samples, but we prefer to check
162 // the flags to catch badly muxed audio (for now anyway ;P). History of
163 // attempts to get this right discussed in http://crrev.com/1319813002
164 bool sample_is_sync_sample
= !(flags
& kSampleIsNonSyncSample
);
165 bool sample_depends_on_others
= sample_depends_on
== kSampleDependsOnOthers
;
166 sample_info
->is_keyframe
= sample_is_sync_sample
&&
167 (!sample_depends_on_others
|| is_audio
);
169 DVLOG(4) << __FUNCTION__
<< " is_kf:" << sample_info
->is_keyframe
170 << " is_sync:" << sample_is_sync_sample
171 << " deps:" << sample_depends_on_others
172 << " audio:" << is_audio
;
177 static const CencSampleEncryptionInfoEntry
* GetSampleEncryptionInfoEntry(
178 const TrackRunInfo
& run_info
,
179 uint32 group_description_index
) {
180 const std::vector
<CencSampleEncryptionInfoEntry
>* entries
= nullptr;
182 // ISO-14496-12 Section 8.9.2.3 and 8.9.4 : group description index
183 // (1) ranges from 1 to the number of sample group entries in the track
184 // level SampleGroupDescription Box, or (2) takes the value 0 to
185 // indicate that this sample is a member of no group, in this case, the
186 // sample is associated with the default values specified in
187 // TrackEncryption Box, or (3) starts at 0x10001, i.e. the index value
188 // 1, with the value 1 in the top 16 bits, to reference fragment-local
189 // SampleGroupDescription Box.
190 // Case (2) is not supported here. The caller must handle it externally
191 // before invoking this function.
192 DCHECK_NE(group_description_index
, 0u);
193 if (group_description_index
>
194 SampleToGroupEntry::kFragmentGroupDescriptionIndexBase
) {
195 group_description_index
-=
196 SampleToGroupEntry::kFragmentGroupDescriptionIndexBase
;
197 entries
= &run_info
.fragment_sample_encryption_info
;
199 entries
= &run_info
.track_sample_encryption_group
->entries
;
202 // |group_description_index| is 1-based.
203 DCHECK_LE(group_description_index
, entries
->size());
204 return (group_description_index
> entries
->size())
206 : &(*entries
)[group_description_index
- 1];
209 // In well-structured encrypted media, each track run will be immediately
210 // preceded by its auxiliary information; this is the only optimal storage
211 // pattern in terms of minimum number of bytes from a serial stream needed to
212 // begin playback. It also allows us to optimize caching on memory-constrained
213 // architectures, because we can cache the relatively small auxiliary
214 // information for an entire run and then discard data from the input stream,
215 // instead of retaining the entire 'mdat' box.
217 // We optimize for this situation (with no loss of generality) by sorting track
218 // runs during iteration in order of their first data offset (either sample data
219 // or auxiliary data).
220 class CompareMinTrackRunDataOffset
{
222 bool operator()(const TrackRunInfo
& a
, const TrackRunInfo
& b
) {
223 int64 a_aux
= a
.aux_info_total_size
? a
.aux_info_start_offset
: kint64max
;
224 int64 b_aux
= b
.aux_info_total_size
? b
.aux_info_start_offset
: kint64max
;
226 int64 a_lesser
= std::min(a_aux
, a
.sample_start_offset
);
227 int64 a_greater
= std::max(a_aux
, a
.sample_start_offset
);
228 int64 b_lesser
= std::min(b_aux
, b
.sample_start_offset
);
229 int64 b_greater
= std::max(b_aux
, b
.sample_start_offset
);
231 if (a_lesser
== b_lesser
) return a_greater
< b_greater
;
232 return a_lesser
< b_lesser
;
236 bool TrackRunIterator::Init(const MovieFragment
& moof
) {
239 for (size_t i
= 0; i
< moof
.tracks
.size(); i
++) {
240 const TrackFragment
& traf
= moof
.tracks
[i
];
242 const Track
* trak
= NULL
;
243 for (size_t t
= 0; t
< moov_
->tracks
.size(); t
++) {
244 if (moov_
->tracks
[t
].header
.track_id
== traf
.header
.track_id
)
245 trak
= &moov_
->tracks
[t
];
249 const TrackExtends
* trex
= NULL
;
250 for (size_t t
= 0; t
< moov_
->extends
.tracks
.size(); t
++) {
251 if (moov_
->extends
.tracks
[t
].track_id
== traf
.header
.track_id
)
252 trex
= &moov_
->extends
.tracks
[t
];
256 const SampleDescription
& stsd
=
257 trak
->media
.information
.sample_table
.description
;
258 if (stsd
.type
!= kAudio
&& stsd
.type
!= kVideo
) {
259 DVLOG(1) << "Skipping unhandled track type";
262 size_t desc_idx
= traf
.header
.sample_description_index
;
263 if (!desc_idx
) desc_idx
= trex
->default_sample_description_index
;
264 RCHECK(desc_idx
> 0); // Descriptions are one-indexed in the file
267 // Process edit list to remove CTS offset introduced in the presence of
268 // B-frames (those that contain a single edit with a nonnegative media
269 // time). Other uses of edit lists are not supported, as they are
270 // both uncommon and better served by higher-level protocols.
271 int64 edit_list_offset
= 0;
272 const std::vector
<EditListEntry
>& edits
= trak
->edit
.list
.edits
;
273 if (!edits
.empty()) {
274 if (edits
.size() > 1)
275 DVLOG(1) << "Multi-entry edit box detected; some components ignored.";
277 if (edits
[0].media_time
< 0) {
278 DVLOG(1) << "Empty edit list entry ignored.";
280 edit_list_offset
= -edits
[0].media_time
;
284 SampleToGroupIterator
sample_to_group_itr(traf
.sample_to_group
);
285 bool is_sample_to_group_valid
= sample_to_group_itr
.IsValid();
287 int64 run_start_dts
= traf
.decode_time
.decode_time
;
288 int sample_count_sum
= 0;
289 for (size_t j
= 0; j
< traf
.runs
.size(); j
++) {
290 const TrackFragmentRun
& trun
= traf
.runs
[j
];
292 tri
.track_id
= traf
.header
.track_id
;
293 tri
.timescale
= trak
->media
.header
.timescale
;
294 tri
.start_dts
= run_start_dts
;
295 tri
.sample_start_offset
= trun
.data_offset
;
296 tri
.track_sample_encryption_group
=
297 &trak
->media
.information
.sample_table
.sample_group_description
;
298 tri
.fragment_sample_encryption_info
=
299 traf
.sample_group_description
.entries
;
301 tri
.is_audio
= (stsd
.type
== kAudio
);
303 RCHECK(!stsd
.audio_entries
.empty());
304 if (desc_idx
> stsd
.audio_entries
.size())
306 tri
.audio_description
= &stsd
.audio_entries
[desc_idx
];
308 RCHECK(!stsd
.video_entries
.empty());
309 if (desc_idx
> stsd
.video_entries
.size())
311 tri
.video_description
= &stsd
.video_entries
[desc_idx
];
314 // Collect information from the auxiliary_offset entry with the same index
315 // in the 'saiz' container as the current run's index in the 'trun'
316 // container, if it is present.
317 if (traf
.auxiliary_offset
.offsets
.size() > j
) {
318 // There should be an auxiliary info entry corresponding to each sample
319 // in the auxiliary offset entry's corresponding track run.
320 RCHECK(traf
.auxiliary_size
.sample_count
>=
321 sample_count_sum
+ trun
.sample_count
);
322 tri
.aux_info_start_offset
= traf
.auxiliary_offset
.offsets
[j
];
323 tri
.aux_info_default_size
=
324 traf
.auxiliary_size
.default_sample_info_size
;
325 if (tri
.aux_info_default_size
== 0) {
326 const std::vector
<uint8
>& sizes
=
327 traf
.auxiliary_size
.sample_info_sizes
;
328 tri
.aux_info_sizes
.insert(tri
.aux_info_sizes
.begin(),
329 sizes
.begin() + sample_count_sum
,
330 sizes
.begin() + sample_count_sum
+ trun
.sample_count
);
333 // If the default info size is positive, find the total size of the aux
334 // info block from it, otherwise sum over the individual sizes of each
335 // aux info entry in the aux_offset entry.
336 if (tri
.aux_info_default_size
) {
337 tri
.aux_info_total_size
=
338 tri
.aux_info_default_size
* trun
.sample_count
;
340 tri
.aux_info_total_size
= 0;
341 for (size_t k
= 0; k
< trun
.sample_count
; k
++) {
342 tri
.aux_info_total_size
+= tri
.aux_info_sizes
[k
];
346 tri
.aux_info_start_offset
= -1;
347 tri
.aux_info_total_size
= 0;
350 tri
.samples
.resize(trun
.sample_count
);
351 for (size_t k
= 0; k
< trun
.sample_count
; k
++) {
352 if (!PopulateSampleInfo(*trex
, traf
.header
, trun
, edit_list_offset
, k
,
353 &tri
.samples
[k
], traf
.sdtp
.sample_depends_on(k
),
354 tri
.is_audio
, media_log_
)) {
358 run_start_dts
+= tri
.samples
[k
].duration
;
360 if (!is_sample_to_group_valid
) {
361 // Set group description index to 0 to read encryption information
362 // from TrackEncryption Box.
363 tri
.samples
[k
].cenc_group_description_index
= 0;
367 uint32 index
= sample_to_group_itr
.group_description_index();
368 tri
.samples
[k
].cenc_group_description_index
= index
;
370 RCHECK(GetSampleEncryptionInfoEntry(tri
, index
));
371 is_sample_to_group_valid
= sample_to_group_itr
.Advance();
373 runs_
.push_back(tri
);
374 sample_count_sum
+= trun
.sample_count
;
377 // We should have iterated through all samples in SampleToGroup Box.
378 RCHECK(!sample_to_group_itr
.IsValid());
381 std::sort(runs_
.begin(), runs_
.end(), CompareMinTrackRunDataOffset());
382 run_itr_
= runs_
.begin();
387 void TrackRunIterator::AdvanceRun() {
392 void TrackRunIterator::ResetRun() {
393 if (!IsRunValid()) return;
394 sample_dts_
= run_itr_
->start_dts
;
395 sample_offset_
= run_itr_
->sample_start_offset
;
396 sample_itr_
= run_itr_
->samples
.begin();
400 void TrackRunIterator::AdvanceSample() {
401 DCHECK(IsSampleValid());
402 sample_dts_
+= sample_itr_
->duration
;
403 sample_offset_
+= sample_itr_
->size
;
407 // This implementation only indicates a need for caching if CENC auxiliary
408 // info is available in the stream.
409 bool TrackRunIterator::AuxInfoNeedsToBeCached() {
410 DCHECK(IsRunValid());
411 return aux_info_size() > 0 && cenc_info_
.size() == 0;
414 // This implementation currently only caches CENC auxiliary info.
415 bool TrackRunIterator::CacheAuxInfo(const uint8
* buf
, int buf_size
) {
416 RCHECK(AuxInfoNeedsToBeCached() && buf_size
>= aux_info_size());
418 cenc_info_
.resize(run_itr_
->samples
.size());
420 for (size_t i
= 0; i
< run_itr_
->samples
.size(); i
++) {
421 int info_size
= run_itr_
->aux_info_default_size
;
423 info_size
= run_itr_
->aux_info_sizes
[i
];
425 if (IsSampleEncrypted(i
)) {
426 BufferReader
reader(buf
+ pos
, info_size
);
427 RCHECK(cenc_info_
[i
].Parse(GetIvSize(i
), &reader
));
435 bool TrackRunIterator::IsRunValid() const {
436 return run_itr_
!= runs_
.end();
439 bool TrackRunIterator::IsSampleValid() const {
440 return IsRunValid() && (sample_itr_
!= run_itr_
->samples
.end());
443 // Because tracks are in sorted order and auxiliary information is cached when
444 // returning samples, it is guaranteed that no data will be required before the
445 // lesser of the minimum data offset of this track and the next in sequence.
446 // (The stronger condition - that no data is required before the minimum data
447 // offset of this track alone - is not guaranteed, because the BMFF spec does
448 // not have any inter-run ordering restrictions.)
449 int64
TrackRunIterator::GetMaxClearOffset() {
450 int64 offset
= kint64max
;
452 if (IsSampleValid()) {
453 offset
= std::min(offset
, sample_offset_
);
454 if (AuxInfoNeedsToBeCached())
455 offset
= std::min(offset
, aux_info_offset());
457 if (run_itr_
!= runs_
.end()) {
458 std::vector
<TrackRunInfo
>::const_iterator next_run
= run_itr_
+ 1;
459 if (next_run
!= runs_
.end()) {
460 offset
= std::min(offset
, next_run
->sample_start_offset
);
461 if (next_run
->aux_info_total_size
)
462 offset
= std::min(offset
, next_run
->aux_info_start_offset
);
465 if (offset
== kint64max
) return 0;
469 uint32
TrackRunIterator::track_id() const {
470 DCHECK(IsRunValid());
471 return run_itr_
->track_id
;
474 bool TrackRunIterator::is_encrypted() const {
475 DCHECK(IsSampleValid());
476 return IsSampleEncrypted(sample_itr_
- run_itr_
->samples
.begin());
479 int64
TrackRunIterator::aux_info_offset() const {
480 return run_itr_
->aux_info_start_offset
;
483 int TrackRunIterator::aux_info_size() const {
484 return run_itr_
->aux_info_total_size
;
487 bool TrackRunIterator::is_audio() const {
488 DCHECK(IsRunValid());
489 return run_itr_
->is_audio
;
492 const AudioSampleEntry
& TrackRunIterator::audio_description() const {
494 DCHECK(run_itr_
->audio_description
);
495 return *run_itr_
->audio_description
;
498 const VideoSampleEntry
& TrackRunIterator::video_description() const {
500 DCHECK(run_itr_
->video_description
);
501 return *run_itr_
->video_description
;
504 int64
TrackRunIterator::sample_offset() const {
505 DCHECK(IsSampleValid());
506 return sample_offset_
;
509 int TrackRunIterator::sample_size() const {
510 DCHECK(IsSampleValid());
511 return sample_itr_
->size
;
514 DecodeTimestamp
TrackRunIterator::dts() const {
515 DCHECK(IsSampleValid());
516 return DecodeTimestampFromRational(sample_dts_
, run_itr_
->timescale
);
519 base::TimeDelta
TrackRunIterator::cts() const {
520 DCHECK(IsSampleValid());
521 return TimeDeltaFromRational(sample_dts_
+ sample_itr_
->cts_offset
,
522 run_itr_
->timescale
);
525 base::TimeDelta
TrackRunIterator::duration() const {
526 DCHECK(IsSampleValid());
527 return TimeDeltaFromRational(sample_itr_
->duration
, run_itr_
->timescale
);
530 bool TrackRunIterator::is_keyframe() const {
531 DCHECK(IsSampleValid());
532 return sample_itr_
->is_keyframe
;
535 const TrackEncryption
& TrackRunIterator::track_encryption() const {
537 return audio_description().sinf
.info
.track_encryption
;
538 return video_description().sinf
.info
.track_encryption
;
541 scoped_ptr
<DecryptConfig
> TrackRunIterator::GetDecryptConfig() {
542 DCHECK(is_encrypted());
544 if (cenc_info_
.empty()) {
545 DCHECK_EQ(0, aux_info_size());
546 MEDIA_LOG(ERROR
, media_log_
) << "Aux Info is not available.";
547 return scoped_ptr
<DecryptConfig
>();
550 size_t sample_idx
= sample_itr_
- run_itr_
->samples
.begin();
551 DCHECK_LT(sample_idx
, cenc_info_
.size());
552 const FrameCENCInfo
& cenc_info
= cenc_info_
[sample_idx
];
554 size_t total_size
= 0;
555 if (!cenc_info
.subsamples
.empty() &&
556 (!cenc_info
.GetTotalSizeOfSubsamples(&total_size
) ||
557 total_size
!= static_cast<size_t>(sample_size()))) {
558 MEDIA_LOG(ERROR
, media_log_
) << "Incorrect CENC subsample size.";
559 return scoped_ptr
<DecryptConfig
>();
562 const std::vector
<uint8
>& kid
= GetKeyId(sample_idx
);
563 return scoped_ptr
<DecryptConfig
>(new DecryptConfig(
564 std::string(reinterpret_cast<const char*>(&kid
[0]), kid
.size()),
565 std::string(reinterpret_cast<const char*>(cenc_info
.iv
),
566 arraysize(cenc_info
.iv
)),
567 cenc_info
.subsamples
));
570 uint32
TrackRunIterator::GetGroupDescriptionIndex(uint32 sample_index
) const {
571 DCHECK(IsRunValid());
572 DCHECK_LT(sample_index
, run_itr_
->samples
.size());
573 return run_itr_
->samples
[sample_index
].cenc_group_description_index
;
576 bool TrackRunIterator::IsSampleEncrypted(size_t sample_index
) const {
577 uint32 index
= GetGroupDescriptionIndex(sample_index
);
579 ? track_encryption().is_encrypted
580 : GetSampleEncryptionInfoEntry(*run_itr_
, index
)->is_encrypted
;
583 const std::vector
<uint8
>& TrackRunIterator::GetKeyId(
584 size_t sample_index
) const {
585 uint32 index
= GetGroupDescriptionIndex(sample_index
);
586 return (index
== 0) ? track_encryption().default_kid
587 : GetSampleEncryptionInfoEntry(*run_itr_
, index
)->key_id
;
590 uint8
TrackRunIterator::GetIvSize(size_t sample_index
) const {
591 uint32 index
= GetGroupDescriptionIndex(sample_index
);
592 return (index
== 0) ? track_encryption().default_iv_size
593 : GetSampleEncryptionInfoEntry(*run_itr_
, index
)->iv_size
;