Delete unused downloads page asset.
[chromium-blink-merge.git] / media / base / audio_splicer.cc
blob3fd6dacbf791994d00e283608c496798e7786de6
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/base/audio_splicer.h"
7 #include <cstdlib>
8 #include <deque>
10 #include "base/logging.h"
11 #include "media/base/audio_buffer.h"
12 #include "media/base/audio_bus.h"
13 #include "media/base/audio_decoder_config.h"
14 #include "media/base/audio_timestamp_helper.h"
15 #include "media/base/media_log.h"
16 #include "media/base/vector_math.h"
18 namespace media {
20 namespace {
22 enum {
23 // Minimum gap size needed before the splicer will take action to
24 // fill a gap. This avoids periodically inserting and then dropping samples
25 // when the buffer timestamps are slightly off because of timestamp rounding
26 // in the source content. Unit is frames.
27 kMinGapSize = 2,
29 // Limits the number of MEDIA_LOG() per sanitizer instance warning the user
30 // about splicer overlaps within |kMaxTimeDeltaInMilliseconds| or gaps larger
31 // than |kMinGapSize| and less than |kMaxTimeDeltaInMilliseconds|. These
32 // warnings may be frequent for some streams, and number of sanitizer
33 // instances may be high, so keep this limit low to help reduce log spam.
34 kMaxSanitizerWarningLogs = 5,
37 // AudioBuffer::TrimStart() is not as accurate as the timestamp helper, so
38 // manually adjust the duration and timestamp after trimming.
39 void AccurateTrimStart(int frames_to_trim,
40 const scoped_refptr<AudioBuffer> buffer,
41 const AudioTimestampHelper& timestamp_helper) {
42 buffer->TrimStart(frames_to_trim);
43 buffer->set_timestamp(timestamp_helper.GetTimestamp());
46 // Returns an AudioBus whose frame buffer is backed by the provided AudioBuffer.
47 scoped_ptr<AudioBus> CreateAudioBufferWrapper(
48 const scoped_refptr<AudioBuffer>& buffer) {
49 scoped_ptr<AudioBus> wrapper =
50 AudioBus::CreateWrapper(buffer->channel_count());
51 wrapper->set_frames(buffer->frame_count());
52 for (int ch = 0; ch < buffer->channel_count(); ++ch) {
53 wrapper->SetChannelData(
54 ch, reinterpret_cast<float*>(buffer->channel_data()[ch]));
56 return wrapper.Pass();
59 } // namespace
61 class AudioStreamSanitizer {
62 public:
63 AudioStreamSanitizer(int samples_per_second,
64 const scoped_refptr<MediaLog>& media_log);
65 ~AudioStreamSanitizer();
67 // Resets the sanitizer state by clearing the output buffers queue, and
68 // resetting the timestamp helper.
69 void Reset();
71 // Similar to Reset(), but initializes the timestamp helper with the given
72 // parameters.
73 void ResetTimestampState(int64 frame_count, base::TimeDelta base_timestamp);
75 // Adds a new buffer full of samples or end of stream buffer to the splicer.
76 // Returns true if the buffer was accepted. False is returned if an error
77 // occurred.
78 bool AddInput(const scoped_refptr<AudioBuffer>& input);
80 // Returns true if the sanitizer has a buffer to return.
81 bool HasNextBuffer() const;
83 // Removes the next buffer from the output buffer queue and returns it; should
84 // only be called if HasNextBuffer() returns true.
85 scoped_refptr<AudioBuffer> GetNextBuffer();
87 // Returns the total frame count of all buffers available for output.
88 int GetFrameCount() const;
90 const AudioTimestampHelper& timestamp_helper() {
91 return output_timestamp_helper_;
94 // Transfer all buffers into |output|. Returns false if AddInput() on the
95 // |output| sanitizer fails for any buffer removed from |this|.
96 bool DrainInto(AudioStreamSanitizer* output);
98 private:
99 void AddOutputBuffer(const scoped_refptr<AudioBuffer>& buffer);
101 AudioTimestampHelper output_timestamp_helper_;
102 bool received_end_of_stream_ = false;
104 typedef std::deque<scoped_refptr<AudioBuffer> > BufferQueue;
105 BufferQueue output_buffers_;
107 scoped_refptr<MediaLog> media_log_;
109 // To prevent log spam, counts the number of audio gap or overlaps warned in
110 // logs.
111 int num_warning_logs_ = 0;
113 DISALLOW_ASSIGN(AudioStreamSanitizer);
116 AudioStreamSanitizer::AudioStreamSanitizer(
117 int samples_per_second,
118 const scoped_refptr<MediaLog>& media_log)
119 : output_timestamp_helper_(samples_per_second), media_log_(media_log) {}
121 AudioStreamSanitizer::~AudioStreamSanitizer() {}
123 void AudioStreamSanitizer::Reset() {
124 ResetTimestampState(0, kNoTimestamp());
127 void AudioStreamSanitizer::ResetTimestampState(int64 frame_count,
128 base::TimeDelta base_timestamp) {
129 output_buffers_.clear();
130 received_end_of_stream_ = false;
131 output_timestamp_helper_.SetBaseTimestamp(base_timestamp);
132 if (frame_count > 0)
133 output_timestamp_helper_.AddFrames(frame_count);
136 bool AudioStreamSanitizer::AddInput(const scoped_refptr<AudioBuffer>& input) {
137 DCHECK(!received_end_of_stream_ || input->end_of_stream());
139 if (input->end_of_stream()) {
140 output_buffers_.push_back(input);
141 received_end_of_stream_ = true;
142 return true;
145 DCHECK(input->timestamp() != kNoTimestamp());
146 DCHECK(input->duration() > base::TimeDelta());
147 DCHECK_GT(input->frame_count(), 0);
149 if (output_timestamp_helper_.base_timestamp() == kNoTimestamp())
150 output_timestamp_helper_.SetBaseTimestamp(input->timestamp());
152 if (output_timestamp_helper_.base_timestamp() > input->timestamp()) {
153 MEDIA_LOG(ERROR, media_log_)
154 << "Audio splicing failed: unexpected timestamp sequence. base "
155 "timestamp="
156 << output_timestamp_helper_.base_timestamp().InMicroseconds()
157 << "us, input timestamp=" << input->timestamp().InMicroseconds()
158 << "us";
159 return false;
162 const base::TimeDelta timestamp = input->timestamp();
163 const base::TimeDelta expected_timestamp =
164 output_timestamp_helper_.GetTimestamp();
165 const base::TimeDelta delta = timestamp - expected_timestamp;
167 if (std::abs(delta.InMilliseconds()) >
168 AudioSplicer::kMaxTimeDeltaInMilliseconds) {
169 MEDIA_LOG(ERROR, media_log_)
170 << "Audio splicing failed: coded frame timestamp differs from "
171 "expected timestamp " << expected_timestamp.InMicroseconds()
172 << "us by " << delta.InMicroseconds()
173 << "us, more than threshold of +/-"
174 << AudioSplicer::kMaxTimeDeltaInMilliseconds
175 << "ms. Expected timestamp is based on decoded frames and frame rate.";
176 return false;
179 int frames_to_fill = 0;
180 if (delta != base::TimeDelta())
181 frames_to_fill = output_timestamp_helper_.GetFramesToTarget(timestamp);
183 if (frames_to_fill == 0 || std::abs(frames_to_fill) < kMinGapSize) {
184 AddOutputBuffer(input);
185 return true;
188 if (frames_to_fill > 0) {
189 LIMITED_MEDIA_LOG(DEBUG, media_log_, num_warning_logs_,
190 kMaxSanitizerWarningLogs)
191 << "Audio splicer inserting silence for small gap of "
192 << delta.InMicroseconds() << "us at time "
193 << expected_timestamp.InMicroseconds() << "us.";
194 DVLOG(1) << "Gap detected @ " << expected_timestamp.InMicroseconds()
195 << " us: " << delta.InMicroseconds() << " us";
197 // Create a buffer with enough silence samples to fill the gap and
198 // add it to the output buffer.
199 scoped_refptr<AudioBuffer> gap =
200 AudioBuffer::CreateEmptyBuffer(input->channel_layout(),
201 input->channel_count(),
202 input->sample_rate(),
203 frames_to_fill,
204 expected_timestamp);
205 AddOutputBuffer(gap);
207 // Add the input buffer now that the gap has been filled.
208 AddOutputBuffer(input);
209 return true;
212 // Overlapping buffers marked as splice frames are handled by AudioSplicer,
213 // but decoder and demuxer quirks may sometimes produce overlapping samples
214 // which need to be sanitized.
216 // A crossfade can't be done here because only the current buffer is available
217 // at this point, not previous buffers.
218 LIMITED_MEDIA_LOG(DEBUG, media_log_, num_warning_logs_,
219 kMaxSanitizerWarningLogs)
220 << "Audio splicer skipping frames for small overlap of "
221 << -delta.InMicroseconds() << "us at time "
222 << expected_timestamp.InMicroseconds() << "us.";
223 DVLOG(1) << "Overlap detected @ " << expected_timestamp.InMicroseconds()
224 << " us: " << -delta.InMicroseconds() << " us";
226 const int frames_to_skip = -frames_to_fill;
227 if (input->frame_count() <= frames_to_skip) {
228 DVLOG(1) << "Dropping whole buffer";
229 return true;
232 // Copy the trailing samples that do not overlap samples already output
233 // into a new buffer. Add this new buffer to the output queue.
235 // TODO(acolwell): Implement a cross-fade here so the transition is less
236 // jarring.
237 AccurateTrimStart(frames_to_skip, input, output_timestamp_helper_);
238 AddOutputBuffer(input);
239 return true;
242 bool AudioStreamSanitizer::HasNextBuffer() const {
243 return !output_buffers_.empty();
246 scoped_refptr<AudioBuffer> AudioStreamSanitizer::GetNextBuffer() {
247 scoped_refptr<AudioBuffer> ret = output_buffers_.front();
248 output_buffers_.pop_front();
249 return ret;
252 void AudioStreamSanitizer::AddOutputBuffer(
253 const scoped_refptr<AudioBuffer>& buffer) {
254 output_timestamp_helper_.AddFrames(buffer->frame_count());
255 output_buffers_.push_back(buffer);
258 int AudioStreamSanitizer::GetFrameCount() const {
259 int frame_count = 0;
260 for (const auto& buffer : output_buffers_)
261 frame_count += buffer->frame_count();
262 return frame_count;
265 bool AudioStreamSanitizer::DrainInto(AudioStreamSanitizer* output) {
266 while (HasNextBuffer()) {
267 if (!output->AddInput(GetNextBuffer()))
268 return false;
270 return true;
273 AudioSplicer::AudioSplicer(int samples_per_second,
274 const scoped_refptr<MediaLog>& media_log)
275 : max_crossfade_duration_(
276 base::TimeDelta::FromMilliseconds(kCrossfadeDurationInMilliseconds)),
277 splice_timestamp_(kNoTimestamp()),
278 max_splice_end_timestamp_(kNoTimestamp()),
279 output_sanitizer_(
280 new AudioStreamSanitizer(samples_per_second, media_log)),
281 pre_splice_sanitizer_(
282 new AudioStreamSanitizer(samples_per_second, media_log)),
283 post_splice_sanitizer_(
284 new AudioStreamSanitizer(samples_per_second, media_log)),
285 have_all_pre_splice_buffers_(false) {
288 AudioSplicer::~AudioSplicer() {}
290 void AudioSplicer::Reset() {
291 output_sanitizer_->Reset();
292 pre_splice_sanitizer_->Reset();
293 post_splice_sanitizer_->Reset();
294 have_all_pre_splice_buffers_ = false;
295 reset_splice_timestamps();
298 bool AudioSplicer::AddInput(const scoped_refptr<AudioBuffer>& input) {
299 // If we're not processing a splice, add the input to the output queue.
300 if (splice_timestamp_ == kNoTimestamp()) {
301 DCHECK(!pre_splice_sanitizer_->HasNextBuffer());
302 DCHECK(!post_splice_sanitizer_->HasNextBuffer());
303 return output_sanitizer_->AddInput(input);
306 const AudioTimestampHelper& output_ts_helper =
307 output_sanitizer_->timestamp_helper();
309 if (!have_all_pre_splice_buffers_) {
310 DCHECK(!input->end_of_stream());
312 // If the provided buffer is entirely before the splice point it can also be
313 // added to the output queue.
314 if (input->timestamp() + input->duration() < splice_timestamp_) {
315 DCHECK(!pre_splice_sanitizer_->HasNextBuffer());
316 return output_sanitizer_->AddInput(input);
319 // If we've encountered the first pre splice buffer, reset the pre splice
320 // sanitizer based on |output_sanitizer_|. This is done so that gaps and
321 // overlaps between buffers across the sanitizers are accounted for prior
322 // to calculating crossfade.
323 if (!pre_splice_sanitizer_->HasNextBuffer()) {
324 pre_splice_sanitizer_->ResetTimestampState(
325 output_ts_helper.frame_count(), output_ts_helper.base_timestamp());
328 return pre_splice_sanitizer_->AddInput(input);
331 // The first post splice buffer is expected to match |splice_timestamp_|.
332 if (!post_splice_sanitizer_->HasNextBuffer())
333 CHECK(splice_timestamp_ == input->timestamp());
335 // At this point we have all the fade out preroll buffers from the decoder.
336 // We now need to wait until we have enough data to perform the crossfade (or
337 // we receive an end of stream).
338 if (!post_splice_sanitizer_->AddInput(input))
339 return false;
341 // Ensure |output_sanitizer_| has a valid base timestamp so we can use it for
342 // timestamp calculations.
343 if (output_ts_helper.base_timestamp() == kNoTimestamp()) {
344 output_sanitizer_->ResetTimestampState(
345 0, pre_splice_sanitizer_->timestamp_helper().base_timestamp());
348 // If a splice frame was incorrectly marked due to poor demuxed timestamps, we
349 // may not actually have a splice. Here we check if any frames exist before
350 // the splice. In this case, just transfer all data to the output sanitizer.
351 const int frames_before_splice =
352 output_ts_helper.GetFramesToTarget(splice_timestamp_);
353 if (frames_before_splice < 0 ||
354 pre_splice_sanitizer_->GetFrameCount() <= frames_before_splice) {
355 CHECK(pre_splice_sanitizer_->DrainInto(output_sanitizer_.get()));
357 // If the file contains incorrectly muxed timestamps, there may be huge gaps
358 // between the demuxed and decoded timestamps.
359 if (!post_splice_sanitizer_->DrainInto(output_sanitizer_.get()))
360 return false;
362 reset_splice_timestamps();
363 return true;
366 // Wait until we have enough data to crossfade or end of stream.
367 if (!input->end_of_stream() &&
368 input->timestamp() + input->duration() < max_splice_end_timestamp_) {
369 return true;
372 scoped_refptr<AudioBuffer> crossfade_buffer;
373 scoped_ptr<AudioBus> pre_splice =
374 ExtractCrossfadeFromPreSplice(&crossfade_buffer);
376 // Crossfade the pre splice and post splice sections and transfer all relevant
377 // buffers into |output_sanitizer_|.
378 CrossfadePostSplice(pre_splice.Pass(), crossfade_buffer);
380 // Clear the splice timestamp so new splices can be accepted.
381 reset_splice_timestamps();
382 return true;
385 bool AudioSplicer::HasNextBuffer() const {
386 return output_sanitizer_->HasNextBuffer();
389 scoped_refptr<AudioBuffer> AudioSplicer::GetNextBuffer() {
390 return output_sanitizer_->GetNextBuffer();
393 void AudioSplicer::SetSpliceTimestamp(base::TimeDelta splice_timestamp) {
394 if (splice_timestamp == kNoTimestamp()) {
395 DCHECK(splice_timestamp_ != kNoTimestamp());
396 DCHECK(!have_all_pre_splice_buffers_);
397 have_all_pre_splice_buffers_ = true;
398 return;
401 if (splice_timestamp_ == splice_timestamp)
402 return;
404 // TODO(dalecurtis): We may need the concept of a future_splice_timestamp_ to
405 // handle cases where another splice comes in before we've received 5ms of
406 // data from the last one. Leave this as a CHECK for now to figure out if
407 // this case is possible.
408 CHECK(splice_timestamp_ == kNoTimestamp());
409 splice_timestamp_ = splice_timestamp;
410 max_splice_end_timestamp_ = splice_timestamp_ + max_crossfade_duration_;
411 pre_splice_sanitizer_->Reset();
412 post_splice_sanitizer_->Reset();
413 have_all_pre_splice_buffers_ = false;
416 scoped_ptr<AudioBus> AudioSplicer::ExtractCrossfadeFromPreSplice(
417 scoped_refptr<AudioBuffer>* crossfade_buffer) {
418 DCHECK(crossfade_buffer);
419 const AudioTimestampHelper& output_ts_helper =
420 output_sanitizer_->timestamp_helper();
422 int frames_before_splice =
423 output_ts_helper.GetFramesToTarget(splice_timestamp_);
425 // Determine crossfade frame count based on available frames in each splicer
426 // and capping to the maximum crossfade duration.
427 const int max_crossfade_frame_count =
428 output_ts_helper.GetFramesToTarget(max_splice_end_timestamp_) -
429 frames_before_splice;
430 const int frames_to_crossfade = std::min(
431 max_crossfade_frame_count,
432 std::min(pre_splice_sanitizer_->GetFrameCount() - frames_before_splice,
433 post_splice_sanitizer_->GetFrameCount()));
434 // There must always be frames to crossfade, otherwise the splice should not
435 // have been generated.
436 DCHECK_GT(frames_to_crossfade, 0);
438 int frames_read = 0;
439 scoped_ptr<AudioBus> output_bus;
440 while (pre_splice_sanitizer_->HasNextBuffer() &&
441 frames_read < frames_to_crossfade) {
442 scoped_refptr<AudioBuffer> preroll = pre_splice_sanitizer_->GetNextBuffer();
444 // We don't know the channel count until we see the first buffer, so wait
445 // until the first buffer to allocate the output AudioBus.
446 if (!output_bus) {
447 output_bus =
448 AudioBus::Create(preroll->channel_count(), frames_to_crossfade);
449 // Allocate output buffer for crossfade.
450 *crossfade_buffer = AudioBuffer::CreateBuffer(kSampleFormatPlanarF32,
451 preroll->channel_layout(),
452 preroll->channel_count(),
453 preroll->sample_rate(),
454 frames_to_crossfade);
457 // There may be enough of a gap introduced during decoding such that an
458 // entire buffer exists before the splice point.
459 if (frames_before_splice >= preroll->frame_count()) {
460 // Adjust the number of frames remaining before the splice. NOTE: This is
461 // safe since |pre_splice_sanitizer_| is a continuation of the timeline in
462 // |output_sanitizer_|. As such we're guaranteed there are no gaps or
463 // overlaps in the timeline between the two sanitizers.
464 frames_before_splice -= preroll->frame_count();
465 CHECK(output_sanitizer_->AddInput(preroll));
466 continue;
469 const int frames_to_read =
470 std::min(preroll->frame_count() - frames_before_splice,
471 output_bus->frames() - frames_read);
472 preroll->ReadFrames(
473 frames_to_read, frames_before_splice, frames_read, output_bus.get());
474 frames_read += frames_to_read;
476 // If only part of the buffer was consumed, trim it appropriately and stick
477 // it into the output queue.
478 if (frames_before_splice) {
479 preroll->TrimEnd(preroll->frame_count() - frames_before_splice);
480 CHECK(output_sanitizer_->AddInput(preroll));
481 frames_before_splice = 0;
485 // Ensure outputs were properly allocated. The method should not have been
486 // called if there is not enough data to crossfade.
487 // TODO(dalecurtis): Convert to DCHECK() once http://crbug.com/356073 fixed.
488 CHECK(output_bus);
489 CHECK(crossfade_buffer->get());
491 // All necessary buffers have been processed, it's safe to reset.
492 pre_splice_sanitizer_->Reset();
493 DCHECK_EQ(output_bus->frames(), frames_read);
494 DCHECK_EQ(output_ts_helper.GetFramesToTarget(splice_timestamp_), 0);
495 return output_bus.Pass();
498 void AudioSplicer::CrossfadePostSplice(
499 scoped_ptr<AudioBus> pre_splice_bus,
500 const scoped_refptr<AudioBuffer>& crossfade_buffer) {
501 // Use the calculated timestamp and duration to ensure there's no extra gaps
502 // or overlaps to process when adding the buffer to |output_sanitizer_|.
503 const AudioTimestampHelper& output_ts_helper =
504 output_sanitizer_->timestamp_helper();
505 crossfade_buffer->set_timestamp(output_ts_helper.GetTimestamp());
507 // AudioBuffer::ReadFrames() only allows output into an AudioBus, so wrap
508 // our AudioBuffer in one so we can avoid extra data copies.
509 scoped_ptr<AudioBus> output_bus = CreateAudioBufferWrapper(crossfade_buffer);
511 // Extract crossfade section from the |post_splice_sanitizer_|.
512 int frames_read = 0, frames_to_trim = 0;
513 scoped_refptr<AudioBuffer> remainder;
514 while (post_splice_sanitizer_->HasNextBuffer() &&
515 frames_read < output_bus->frames()) {
516 scoped_refptr<AudioBuffer> postroll =
517 post_splice_sanitizer_->GetNextBuffer();
518 const int frames_to_read =
519 std::min(postroll->frame_count(), output_bus->frames() - frames_read);
520 postroll->ReadFrames(frames_to_read, 0, frames_read, output_bus.get());
521 frames_read += frames_to_read;
523 // If only part of the buffer was consumed, save it for after we've added
524 // the crossfade buffer
525 if (frames_to_read < postroll->frame_count()) {
526 DCHECK(!remainder.get());
527 remainder.swap(postroll);
528 frames_to_trim = frames_to_read;
532 DCHECK_EQ(output_bus->frames(), frames_read);
534 // Crossfade the audio into |crossfade_buffer|.
535 for (int ch = 0; ch < output_bus->channels(); ++ch) {
536 vector_math::Crossfade(pre_splice_bus->channel(ch),
537 pre_splice_bus->frames(),
538 output_bus->channel(ch));
541 CHECK(output_sanitizer_->AddInput(crossfade_buffer));
542 DCHECK_EQ(crossfade_buffer->frame_count(), output_bus->frames());
544 if (remainder.get()) {
545 // Trim off consumed frames.
546 AccurateTrimStart(frames_to_trim, remainder, output_ts_helper);
547 CHECK(output_sanitizer_->AddInput(remainder));
550 // Transfer all remaining buffers out and reset once empty.
551 CHECK(post_splice_sanitizer_->DrainInto(output_sanitizer_.get()));
552 post_splice_sanitizer_->Reset();
555 } // namespace media