1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/formats/mp4/track_run_iterator.h"
10 #include "media/base/buffers.h"
11 #include "media/formats/mp4/rcheck.h"
12 #include "media/formats/mp4/sample_to_group_iterator.h"
22 uint32 cenc_group_description_index
;
27 std::vector
<SampleInfo
> samples
;
30 int64 sample_start_offset
;
33 const AudioSampleEntry
* audio_description
;
34 const VideoSampleEntry
* video_description
;
35 const SampleGroupDescription
* track_sample_encryption_group
;
37 int64 aux_info_start_offset
; // Only valid if aux_info_total_size > 0.
38 int aux_info_default_size
;
39 std::vector
<uint8
> aux_info_sizes
; // Populated if default_size == 0.
40 int aux_info_total_size
;
42 std::vector
<CencSampleEncryptionInfoEntry
> fragment_sample_encryption_info
;
48 TrackRunInfo::TrackRunInfo()
52 sample_start_offset(-1),
54 aux_info_start_offset(-1),
55 aux_info_default_size(-1),
56 aux_info_total_size(-1) {
58 TrackRunInfo::~TrackRunInfo() {}
60 base::TimeDelta
TimeDeltaFromRational(int64 numer
, int64 denom
) {
61 // To avoid overflow, split the following calculation:
62 // (numer * base::Time::kMicrosecondsPerSecond) / denom
64 // (numer / denom) * base::Time::kMicrosecondsPerSecond +
65 // ((numer % denom) * base::Time::kMicrosecondsPerSecond) / denom
66 int64 a
= numer
/ denom
;
67 DCHECK_LE((a
> 0 ? a
: -a
), kint64max
/ base::Time::kMicrosecondsPerSecond
);
68 int64 timea_in_us
= a
* base::Time::kMicrosecondsPerSecond
;
70 int64 b
= numer
% denom
;
71 DCHECK_LE((b
> 0 ? b
: -b
), kint64max
/ base::Time::kMicrosecondsPerSecond
);
72 int64 timeb_in_us
= (b
* base::Time::kMicrosecondsPerSecond
) / denom
;
74 DCHECK((timeb_in_us
< 0) || (timea_in_us
<= kint64max
- timeb_in_us
));
75 DCHECK((timeb_in_us
> 0) || (timea_in_us
>= kint64min
- timeb_in_us
));
76 return base::TimeDelta::FromMicroseconds(timea_in_us
+ timeb_in_us
);
79 DecodeTimestamp
DecodeTimestampFromRational(int64 numer
, int64 denom
) {
80 return DecodeTimestamp::FromPresentationTime(
81 TimeDeltaFromRational(numer
, denom
));
84 TrackRunIterator::TrackRunIterator(const Movie
* moov
,
85 const scoped_refptr
<MediaLog
>& media_log
)
86 : moov_(moov
), media_log_(media_log
), sample_offset_(0) {
90 TrackRunIterator::~TrackRunIterator() {}
92 static std::string
HexFlags(uint32 flags
) {
93 std::stringstream stream
;
94 stream
<< std::setfill('0') << std::setw(sizeof(flags
)*2) << std::hex
99 static bool PopulateSampleInfo(const TrackExtends
& trex
,
100 const TrackFragmentHeader
& tfhd
,
101 const TrackFragmentRun
& trun
,
102 const int64 edit_list_offset
,
104 SampleInfo
* sample_info
,
105 const SampleDependsOn sdtp_sample_depends_on
,
107 const scoped_refptr
<MediaLog
>& media_log
) {
108 if (i
< trun
.sample_sizes
.size()) {
109 sample_info
->size
= trun
.sample_sizes
[i
];
110 } else if (tfhd
.default_sample_size
> 0) {
111 sample_info
->size
= tfhd
.default_sample_size
;
113 sample_info
->size
= trex
.default_sample_size
;
116 if (i
< trun
.sample_durations
.size()) {
117 sample_info
->duration
= trun
.sample_durations
[i
];
118 } else if (tfhd
.default_sample_duration
> 0) {
119 sample_info
->duration
= tfhd
.default_sample_duration
;
121 sample_info
->duration
= trex
.default_sample_duration
;
124 if (i
< trun
.sample_composition_time_offsets
.size()) {
125 sample_info
->cts_offset
= trun
.sample_composition_time_offsets
[i
];
127 sample_info
->cts_offset
= 0;
129 sample_info
->cts_offset
+= edit_list_offset
;
132 if (i
< trun
.sample_flags
.size()) {
133 flags
= trun
.sample_flags
[i
];
134 DVLOG(4) << __FUNCTION__
<< " trun sample flags " << HexFlags(flags
);
135 } else if (tfhd
.has_default_sample_flags
) {
136 flags
= tfhd
.default_sample_flags
;
137 DVLOG(4) << __FUNCTION__
<< " tfhd sample flags " << HexFlags(flags
);
139 flags
= trex
.default_sample_flags
;
140 DVLOG(4) << __FUNCTION__
<< " trex sample flags " << HexFlags(flags
);
143 SampleDependsOn sample_depends_on
=
144 static_cast<SampleDependsOn
>((flags
>> 24) & 0x3);
145 if (sample_depends_on
== kSampleDependsOnUnknown
) {
146 sample_depends_on
= sdtp_sample_depends_on
;
148 DVLOG(4) << __FUNCTION__
<< " sample_depends_on " << sample_depends_on
;
149 if (sample_depends_on
== kSampleDependsOnReserved
) {
150 MEDIA_LOG(ERROR
, media_log
) << "Reserved value used in sample dependency"
155 // Per spec (ISO 14496-12:2012), the definition for a "sync sample" is
156 // equivalent to the downstream code's "is keyframe" concept. But media exists
157 // that marks non-key video frames as sync samples (http://crbug.com/507916
158 // and http://crbug.com/310712). Hence, for video we additionally check that
159 // the sample does not depend on others (FFmpeg does too, see mov_read_trun).
160 // Sample dependency is not ignored for audio because encoded audio samples
161 // can depend on other samples and still be used for random access. Generally
162 // all audio samples are expected to be sync samples, but we prefer to check
163 // the flags to catch badly muxed audio (for now anyway ;P). History of
164 // attempts to get this right discussed in http://crrev.com/1319813002
165 bool sample_is_sync_sample
= !(flags
& kSampleIsNonSyncSample
);
166 bool sample_depends_on_others
= sample_depends_on
== kSampleDependsOnOthers
;
167 sample_info
->is_keyframe
= sample_is_sync_sample
&&
168 (!sample_depends_on_others
|| is_audio
);
170 DVLOG(4) << __FUNCTION__
<< " is_kf:" << sample_info
->is_keyframe
171 << " is_sync:" << sample_is_sync_sample
172 << " deps:" << sample_depends_on_others
173 << " audio:" << is_audio
;
178 static const CencSampleEncryptionInfoEntry
* GetSampleEncryptionInfoEntry(
179 const TrackRunInfo
& run_info
,
180 uint32 group_description_index
) {
181 const std::vector
<CencSampleEncryptionInfoEntry
>* entries
= nullptr;
183 // ISO-14496-12 Section 8.9.2.3 and 8.9.4 : group description index
184 // (1) ranges from 1 to the number of sample group entries in the track
185 // level SampleGroupDescription Box, or (2) takes the value 0 to
186 // indicate that this sample is a member of no group, in this case, the
187 // sample is associated with the default values specified in
188 // TrackEncryption Box, or (3) starts at 0x10001, i.e. the index value
189 // 1, with the value 1 in the top 16 bits, to reference fragment-local
190 // SampleGroupDescription Box.
191 // Case (2) is not supported here. The caller must handle it externally
192 // before invoking this function.
193 DCHECK_NE(group_description_index
, 0u);
194 if (group_description_index
>
195 SampleToGroupEntry::kFragmentGroupDescriptionIndexBase
) {
196 group_description_index
-=
197 SampleToGroupEntry::kFragmentGroupDescriptionIndexBase
;
198 entries
= &run_info
.fragment_sample_encryption_info
;
200 entries
= &run_info
.track_sample_encryption_group
->entries
;
203 // |group_description_index| is 1-based.
204 DCHECK_LE(group_description_index
, entries
->size());
205 return (group_description_index
> entries
->size())
207 : &(*entries
)[group_description_index
- 1];
210 // In well-structured encrypted media, each track run will be immediately
211 // preceded by its auxiliary information; this is the only optimal storage
212 // pattern in terms of minimum number of bytes from a serial stream needed to
213 // begin playback. It also allows us to optimize caching on memory-constrained
214 // architectures, because we can cache the relatively small auxiliary
215 // information for an entire run and then discard data from the input stream,
216 // instead of retaining the entire 'mdat' box.
218 // We optimize for this situation (with no loss of generality) by sorting track
219 // runs during iteration in order of their first data offset (either sample data
220 // or auxiliary data).
221 class CompareMinTrackRunDataOffset
{
223 bool operator()(const TrackRunInfo
& a
, const TrackRunInfo
& b
) {
224 int64 a_aux
= a
.aux_info_total_size
? a
.aux_info_start_offset
: kint64max
;
225 int64 b_aux
= b
.aux_info_total_size
? b
.aux_info_start_offset
: kint64max
;
227 int64 a_lesser
= std::min(a_aux
, a
.sample_start_offset
);
228 int64 a_greater
= std::max(a_aux
, a
.sample_start_offset
);
229 int64 b_lesser
= std::min(b_aux
, b
.sample_start_offset
);
230 int64 b_greater
= std::max(b_aux
, b
.sample_start_offset
);
232 if (a_lesser
== b_lesser
) return a_greater
< b_greater
;
233 return a_lesser
< b_lesser
;
237 bool TrackRunIterator::Init(const MovieFragment
& moof
) {
240 for (size_t i
= 0; i
< moof
.tracks
.size(); i
++) {
241 const TrackFragment
& traf
= moof
.tracks
[i
];
243 const Track
* trak
= NULL
;
244 for (size_t t
= 0; t
< moov_
->tracks
.size(); t
++) {
245 if (moov_
->tracks
[t
].header
.track_id
== traf
.header
.track_id
)
246 trak
= &moov_
->tracks
[t
];
250 const TrackExtends
* trex
= NULL
;
251 for (size_t t
= 0; t
< moov_
->extends
.tracks
.size(); t
++) {
252 if (moov_
->extends
.tracks
[t
].track_id
== traf
.header
.track_id
)
253 trex
= &moov_
->extends
.tracks
[t
];
257 const SampleDescription
& stsd
=
258 trak
->media
.information
.sample_table
.description
;
259 if (stsd
.type
!= kAudio
&& stsd
.type
!= kVideo
) {
260 DVLOG(1) << "Skipping unhandled track type";
263 size_t desc_idx
= traf
.header
.sample_description_index
;
264 if (!desc_idx
) desc_idx
= trex
->default_sample_description_index
;
265 RCHECK(desc_idx
> 0); // Descriptions are one-indexed in the file
268 // Process edit list to remove CTS offset introduced in the presence of
269 // B-frames (those that contain a single edit with a nonnegative media
270 // time). Other uses of edit lists are not supported, as they are
271 // both uncommon and better served by higher-level protocols.
272 int64 edit_list_offset
= 0;
273 const std::vector
<EditListEntry
>& edits
= trak
->edit
.list
.edits
;
274 if (!edits
.empty()) {
275 if (edits
.size() > 1)
276 DVLOG(1) << "Multi-entry edit box detected; some components ignored.";
278 if (edits
[0].media_time
< 0) {
279 DVLOG(1) << "Empty edit list entry ignored.";
281 edit_list_offset
= -edits
[0].media_time
;
285 SampleToGroupIterator
sample_to_group_itr(traf
.sample_to_group
);
286 bool is_sample_to_group_valid
= sample_to_group_itr
.IsValid();
288 int64 run_start_dts
= traf
.decode_time
.decode_time
;
289 int sample_count_sum
= 0;
290 for (size_t j
= 0; j
< traf
.runs
.size(); j
++) {
291 const TrackFragmentRun
& trun
= traf
.runs
[j
];
293 tri
.track_id
= traf
.header
.track_id
;
294 tri
.timescale
= trak
->media
.header
.timescale
;
295 tri
.start_dts
= run_start_dts
;
296 tri
.sample_start_offset
= trun
.data_offset
;
297 tri
.track_sample_encryption_group
=
298 &trak
->media
.information
.sample_table
.sample_group_description
;
299 tri
.fragment_sample_encryption_info
=
300 traf
.sample_group_description
.entries
;
302 tri
.is_audio
= (stsd
.type
== kAudio
);
304 RCHECK(!stsd
.audio_entries
.empty());
305 if (desc_idx
> stsd
.audio_entries
.size())
307 tri
.audio_description
= &stsd
.audio_entries
[desc_idx
];
309 RCHECK(!stsd
.video_entries
.empty());
310 if (desc_idx
> stsd
.video_entries
.size())
312 tri
.video_description
= &stsd
.video_entries
[desc_idx
];
315 // Collect information from the auxiliary_offset entry with the same index
316 // in the 'saiz' container as the current run's index in the 'trun'
317 // container, if it is present.
318 if (traf
.auxiliary_offset
.offsets
.size() > j
) {
319 // There should be an auxiliary info entry corresponding to each sample
320 // in the auxiliary offset entry's corresponding track run.
321 RCHECK(traf
.auxiliary_size
.sample_count
>=
322 sample_count_sum
+ trun
.sample_count
);
323 tri
.aux_info_start_offset
= traf
.auxiliary_offset
.offsets
[j
];
324 tri
.aux_info_default_size
=
325 traf
.auxiliary_size
.default_sample_info_size
;
326 if (tri
.aux_info_default_size
== 0) {
327 const std::vector
<uint8
>& sizes
=
328 traf
.auxiliary_size
.sample_info_sizes
;
329 tri
.aux_info_sizes
.insert(tri
.aux_info_sizes
.begin(),
330 sizes
.begin() + sample_count_sum
,
331 sizes
.begin() + sample_count_sum
+ trun
.sample_count
);
334 // If the default info size is positive, find the total size of the aux
335 // info block from it, otherwise sum over the individual sizes of each
336 // aux info entry in the aux_offset entry.
337 if (tri
.aux_info_default_size
) {
338 tri
.aux_info_total_size
=
339 tri
.aux_info_default_size
* trun
.sample_count
;
341 tri
.aux_info_total_size
= 0;
342 for (size_t k
= 0; k
< trun
.sample_count
; k
++) {
343 tri
.aux_info_total_size
+= tri
.aux_info_sizes
[k
];
347 tri
.aux_info_start_offset
= -1;
348 tri
.aux_info_total_size
= 0;
351 tri
.samples
.resize(trun
.sample_count
);
352 for (size_t k
= 0; k
< trun
.sample_count
; k
++) {
353 if (!PopulateSampleInfo(*trex
, traf
.header
, trun
, edit_list_offset
, k
,
354 &tri
.samples
[k
], traf
.sdtp
.sample_depends_on(k
),
355 tri
.is_audio
, media_log_
)) {
359 run_start_dts
+= tri
.samples
[k
].duration
;
361 if (!is_sample_to_group_valid
) {
362 // Set group description index to 0 to read encryption information
363 // from TrackEncryption Box.
364 tri
.samples
[k
].cenc_group_description_index
= 0;
368 uint32 index
= sample_to_group_itr
.group_description_index();
369 tri
.samples
[k
].cenc_group_description_index
= index
;
371 RCHECK(GetSampleEncryptionInfoEntry(tri
, index
));
372 is_sample_to_group_valid
= sample_to_group_itr
.Advance();
374 runs_
.push_back(tri
);
375 sample_count_sum
+= trun
.sample_count
;
378 // We should have iterated through all samples in SampleToGroup Box.
379 RCHECK(!sample_to_group_itr
.IsValid());
382 std::sort(runs_
.begin(), runs_
.end(), CompareMinTrackRunDataOffset());
383 run_itr_
= runs_
.begin();
388 void TrackRunIterator::AdvanceRun() {
393 void TrackRunIterator::ResetRun() {
394 if (!IsRunValid()) return;
395 sample_dts_
= run_itr_
->start_dts
;
396 sample_offset_
= run_itr_
->sample_start_offset
;
397 sample_itr_
= run_itr_
->samples
.begin();
401 void TrackRunIterator::AdvanceSample() {
402 DCHECK(IsSampleValid());
403 sample_dts_
+= sample_itr_
->duration
;
404 sample_offset_
+= sample_itr_
->size
;
408 // This implementation only indicates a need for caching if CENC auxiliary
409 // info is available in the stream.
410 bool TrackRunIterator::AuxInfoNeedsToBeCached() {
411 DCHECK(IsRunValid());
412 return aux_info_size() > 0 && cenc_info_
.size() == 0;
415 // This implementation currently only caches CENC auxiliary info.
416 bool TrackRunIterator::CacheAuxInfo(const uint8
* buf
, int buf_size
) {
417 RCHECK(AuxInfoNeedsToBeCached() && buf_size
>= aux_info_size());
419 cenc_info_
.resize(run_itr_
->samples
.size());
421 for (size_t i
= 0; i
< run_itr_
->samples
.size(); i
++) {
422 int info_size
= run_itr_
->aux_info_default_size
;
424 info_size
= run_itr_
->aux_info_sizes
[i
];
426 if (IsSampleEncrypted(i
)) {
427 BufferReader
reader(buf
+ pos
, info_size
);
428 RCHECK(cenc_info_
[i
].Parse(GetIvSize(i
), &reader
));
436 bool TrackRunIterator::IsRunValid() const {
437 return run_itr_
!= runs_
.end();
440 bool TrackRunIterator::IsSampleValid() const {
441 return IsRunValid() && (sample_itr_
!= run_itr_
->samples
.end());
444 // Because tracks are in sorted order and auxiliary information is cached when
445 // returning samples, it is guaranteed that no data will be required before the
446 // lesser of the minimum data offset of this track and the next in sequence.
447 // (The stronger condition - that no data is required before the minimum data
448 // offset of this track alone - is not guaranteed, because the BMFF spec does
449 // not have any inter-run ordering restrictions.)
450 int64
TrackRunIterator::GetMaxClearOffset() {
451 int64 offset
= kint64max
;
453 if (IsSampleValid()) {
454 offset
= std::min(offset
, sample_offset_
);
455 if (AuxInfoNeedsToBeCached())
456 offset
= std::min(offset
, aux_info_offset());
458 if (run_itr_
!= runs_
.end()) {
459 std::vector
<TrackRunInfo
>::const_iterator next_run
= run_itr_
+ 1;
460 if (next_run
!= runs_
.end()) {
461 offset
= std::min(offset
, next_run
->sample_start_offset
);
462 if (next_run
->aux_info_total_size
)
463 offset
= std::min(offset
, next_run
->aux_info_start_offset
);
466 if (offset
== kint64max
) return 0;
470 uint32
TrackRunIterator::track_id() const {
471 DCHECK(IsRunValid());
472 return run_itr_
->track_id
;
475 bool TrackRunIterator::is_encrypted() const {
476 DCHECK(IsSampleValid());
477 return IsSampleEncrypted(sample_itr_
- run_itr_
->samples
.begin());
480 int64
TrackRunIterator::aux_info_offset() const {
481 return run_itr_
->aux_info_start_offset
;
484 int TrackRunIterator::aux_info_size() const {
485 return run_itr_
->aux_info_total_size
;
488 bool TrackRunIterator::is_audio() const {
489 DCHECK(IsRunValid());
490 return run_itr_
->is_audio
;
493 const AudioSampleEntry
& TrackRunIterator::audio_description() const {
495 DCHECK(run_itr_
->audio_description
);
496 return *run_itr_
->audio_description
;
499 const VideoSampleEntry
& TrackRunIterator::video_description() const {
501 DCHECK(run_itr_
->video_description
);
502 return *run_itr_
->video_description
;
505 int64
TrackRunIterator::sample_offset() const {
506 DCHECK(IsSampleValid());
507 return sample_offset_
;
510 int TrackRunIterator::sample_size() const {
511 DCHECK(IsSampleValid());
512 return sample_itr_
->size
;
515 DecodeTimestamp
TrackRunIterator::dts() const {
516 DCHECK(IsSampleValid());
517 return DecodeTimestampFromRational(sample_dts_
, run_itr_
->timescale
);
520 base::TimeDelta
TrackRunIterator::cts() const {
521 DCHECK(IsSampleValid());
522 return TimeDeltaFromRational(sample_dts_
+ sample_itr_
->cts_offset
,
523 run_itr_
->timescale
);
526 base::TimeDelta
TrackRunIterator::duration() const {
527 DCHECK(IsSampleValid());
528 return TimeDeltaFromRational(sample_itr_
->duration
, run_itr_
->timescale
);
531 bool TrackRunIterator::is_keyframe() const {
532 DCHECK(IsSampleValid());
533 return sample_itr_
->is_keyframe
;
536 const TrackEncryption
& TrackRunIterator::track_encryption() const {
538 return audio_description().sinf
.info
.track_encryption
;
539 return video_description().sinf
.info
.track_encryption
;
542 scoped_ptr
<DecryptConfig
> TrackRunIterator::GetDecryptConfig() {
543 DCHECK(is_encrypted());
545 if (cenc_info_
.empty()) {
546 DCHECK_EQ(0, aux_info_size());
547 MEDIA_LOG(ERROR
, media_log_
) << "Aux Info is not available.";
548 return scoped_ptr
<DecryptConfig
>();
551 size_t sample_idx
= sample_itr_
- run_itr_
->samples
.begin();
552 DCHECK_LT(sample_idx
, cenc_info_
.size());
553 const FrameCENCInfo
& cenc_info
= cenc_info_
[sample_idx
];
555 size_t total_size
= 0;
556 if (!cenc_info
.subsamples
.empty() &&
557 (!cenc_info
.GetTotalSizeOfSubsamples(&total_size
) ||
558 total_size
!= static_cast<size_t>(sample_size()))) {
559 MEDIA_LOG(ERROR
, media_log_
) << "Incorrect CENC subsample size.";
560 return scoped_ptr
<DecryptConfig
>();
563 const std::vector
<uint8
>& kid
= GetKeyId(sample_idx
);
564 return scoped_ptr
<DecryptConfig
>(new DecryptConfig(
565 std::string(reinterpret_cast<const char*>(&kid
[0]), kid
.size()),
566 std::string(reinterpret_cast<const char*>(cenc_info
.iv
),
567 arraysize(cenc_info
.iv
)),
568 cenc_info
.subsamples
));
571 uint32
TrackRunIterator::GetGroupDescriptionIndex(uint32 sample_index
) const {
572 DCHECK(IsRunValid());
573 DCHECK_LT(sample_index
, run_itr_
->samples
.size());
574 return run_itr_
->samples
[sample_index
].cenc_group_description_index
;
577 bool TrackRunIterator::IsSampleEncrypted(size_t sample_index
) const {
578 uint32 index
= GetGroupDescriptionIndex(sample_index
);
580 ? track_encryption().is_encrypted
581 : GetSampleEncryptionInfoEntry(*run_itr_
, index
)->is_encrypted
;
584 const std::vector
<uint8
>& TrackRunIterator::GetKeyId(
585 size_t sample_index
) const {
586 uint32 index
= GetGroupDescriptionIndex(sample_index
);
587 return (index
== 0) ? track_encryption().default_kid
588 : GetSampleEncryptionInfoEntry(*run_itr_
, index
)->key_id
;
591 uint8
TrackRunIterator::GetIvSize(size_t sample_index
) const {
592 uint32 index
= GetGroupDescriptionIndex(sample_index
);
593 return (index
== 0) ? track_encryption().default_iv_size
594 : GetSampleEncryptionInfoEntry(*run_itr_
, index
)->iv_size
;