1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/audio_renderer_impl.h"
11 #include "base/bind.h"
12 #include "base/callback.h"
13 #include "base/callback_helpers.h"
14 #include "base/logging.h"
15 #include "base/metrics/histogram.h"
16 #include "base/single_thread_task_runner.h"
17 #include "media/base/audio_buffer.h"
18 #include "media/base/audio_buffer_converter.h"
19 #include "media/base/audio_hardware_config.h"
20 #include "media/base/audio_splicer.h"
21 #include "media/base/bind_to_current_loop.h"
22 #include "media/base/demuxer_stream.h"
23 #include "media/filters/audio_clock.h"
24 #include "media/filters/decrypting_demuxer_stream.h"
30 enum AudioRendererEvent
{
33 RENDER_EVENT_MAX
= RENDER_ERROR
,
36 void HistogramRendererEvent(AudioRendererEvent event
) {
37 UMA_HISTOGRAM_ENUMERATION(
38 "Media.AudioRendererEvents", event
, RENDER_EVENT_MAX
+ 1);
43 AudioRendererImpl::AudioRendererImpl(
44 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
,
45 media::AudioRendererSink
* sink
,
46 ScopedVector
<AudioDecoder
> decoders
,
47 const SetDecryptorReadyCB
& set_decryptor_ready_cb
,
48 const AudioHardwareConfig
& hardware_config
,
49 const scoped_refptr
<MediaLog
>& media_log
)
50 : task_runner_(task_runner
),
51 expecting_config_changes_(false),
53 audio_buffer_stream_(new AudioBufferStream(task_runner
,
55 set_decryptor_ready_cb
,
57 hardware_config_(hardware_config
),
59 state_(kUninitialized
),
60 buffering_state_(BUFFERING_HAVE_NOTHING
),
64 received_end_of_stream_(false),
65 rendered_end_of_stream_(false),
67 audio_buffer_stream_
->set_splice_observer(base::Bind(
68 &AudioRendererImpl::OnNewSpliceBuffer
, weak_factory_
.GetWeakPtr()));
69 audio_buffer_stream_
->set_config_change_observer(base::Bind(
70 &AudioRendererImpl::OnConfigChange
, weak_factory_
.GetWeakPtr()));
73 AudioRendererImpl::~AudioRendererImpl() {
74 DVLOG(1) << __FUNCTION__
;
75 DCHECK(task_runner_
->BelongsToCurrentThread());
77 // If Render() is in progress, this call will wait for Render() to finish.
78 // After this call, the |sink_| will not call back into |this| anymore.
81 if (!init_cb_
.is_null())
82 base::ResetAndReturn(&init_cb_
).Run(PIPELINE_ERROR_ABORT
);
85 void AudioRendererImpl::StartTicking() {
86 DVLOG(1) << __FUNCTION__
;
87 DCHECK(task_runner_
->BelongsToCurrentThread());
91 base::AutoLock
auto_lock(lock_
);
92 // Wait for an eventual call to SetPlaybackRate() to start rendering.
93 if (playback_rate_
== 0) {
94 DCHECK(!sink_playing_
);
98 StartRendering_Locked();
101 void AudioRendererImpl::StartRendering_Locked() {
102 DVLOG(1) << __FUNCTION__
;
103 DCHECK(task_runner_
->BelongsToCurrentThread());
104 DCHECK_EQ(state_
, kPlaying
);
105 DCHECK(!sink_playing_
);
106 DCHECK_NE(playback_rate_
, 0);
107 lock_
.AssertAcquired();
109 sink_playing_
= true;
111 base::AutoUnlock
auto_unlock(lock_
);
115 void AudioRendererImpl::StopTicking() {
116 DVLOG(1) << __FUNCTION__
;
117 DCHECK(task_runner_
->BelongsToCurrentThread());
121 base::AutoLock
auto_lock(lock_
);
122 // Rendering should have already been stopped with a zero playback rate.
123 if (playback_rate_
== 0) {
124 DCHECK(!sink_playing_
);
128 StopRendering_Locked();
131 void AudioRendererImpl::StopRendering_Locked() {
132 DCHECK(task_runner_
->BelongsToCurrentThread());
133 DCHECK_EQ(state_
, kPlaying
);
134 DCHECK(sink_playing_
);
135 lock_
.AssertAcquired();
137 sink_playing_
= false;
139 base::AutoUnlock
auto_unlock(lock_
);
143 void AudioRendererImpl::SetMediaTime(base::TimeDelta time
) {
144 DVLOG(1) << __FUNCTION__
<< "(" << time
.InMicroseconds() << ")";
145 DCHECK(task_runner_
->BelongsToCurrentThread());
147 base::AutoLock
auto_lock(lock_
);
149 DCHECK_EQ(state_
, kFlushed
);
151 start_timestamp_
= time
;
152 ended_timestamp_
= kInfiniteDuration();
153 last_render_ticks_
= base::TimeTicks();
154 audio_clock_
.reset(new AudioClock(time
, audio_parameters_
.sample_rate()));
157 base::TimeDelta
AudioRendererImpl::CurrentMediaTime() {
158 // In practice the Render() method is called with a high enough frequency
159 // that returning only the front timestamp is good enough and also prevents
160 // returning values that go backwards in time.
161 base::TimeDelta current_media_time
;
163 base::AutoLock
auto_lock(lock_
);
164 current_media_time
= audio_clock_
->front_timestamp();
167 DVLOG(3) << __FUNCTION__
<< ": " << current_media_time
.InMilliseconds()
169 return current_media_time
;
172 base::TimeDelta
AudioRendererImpl::CurrentMediaTimeForSyncingVideo() {
173 DVLOG(2) << __FUNCTION__
;
175 base::AutoLock
auto_lock(lock_
);
176 if (last_render_ticks_
.is_null())
177 return audio_clock_
->front_timestamp();
179 return audio_clock_
->TimestampSinceWriting(base::TimeTicks::Now() -
183 TimeSource
* AudioRendererImpl::GetTimeSource() {
187 void AudioRendererImpl::Flush(const base::Closure
& callback
) {
188 DVLOG(1) << __FUNCTION__
;
189 DCHECK(task_runner_
->BelongsToCurrentThread());
191 base::AutoLock
auto_lock(lock_
);
192 DCHECK_EQ(state_
, kPlaying
);
193 DCHECK(flush_cb_
.is_null());
195 flush_cb_
= callback
;
196 ChangeState_Locked(kFlushing
);
201 ChangeState_Locked(kFlushed
);
205 void AudioRendererImpl::DoFlush_Locked() {
206 DCHECK(task_runner_
->BelongsToCurrentThread());
207 lock_
.AssertAcquired();
209 DCHECK(!pending_read_
);
210 DCHECK_EQ(state_
, kFlushed
);
212 audio_buffer_stream_
->Reset(base::Bind(&AudioRendererImpl::ResetDecoderDone
,
213 weak_factory_
.GetWeakPtr()));
216 void AudioRendererImpl::ResetDecoderDone() {
217 DCHECK(task_runner_
->BelongsToCurrentThread());
219 base::AutoLock
auto_lock(lock_
);
221 DCHECK_EQ(state_
, kFlushed
);
222 DCHECK(!flush_cb_
.is_null());
224 received_end_of_stream_
= false;
225 rendered_end_of_stream_
= false;
227 // Flush() may have been called while underflowed/not fully buffered.
228 if (buffering_state_
!= BUFFERING_HAVE_NOTHING
)
229 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING
);
232 if (buffer_converter_
)
233 buffer_converter_
->Reset();
234 algorithm_
->FlushBuffers();
237 // Changes in buffering state are always posted. Flush callback must only be
238 // run after buffering state has been set back to nothing.
239 task_runner_
->PostTask(FROM_HERE
, base::ResetAndReturn(&flush_cb_
));
242 void AudioRendererImpl::StartPlaying() {
243 DVLOG(1) << __FUNCTION__
;
244 DCHECK(task_runner_
->BelongsToCurrentThread());
246 base::AutoLock
auto_lock(lock_
);
247 DCHECK(!sink_playing_
);
248 DCHECK_EQ(state_
, kFlushed
);
249 DCHECK_EQ(buffering_state_
, BUFFERING_HAVE_NOTHING
);
250 DCHECK(!pending_read_
) << "Pending read must complete before seeking";
252 ChangeState_Locked(kPlaying
);
253 AttemptRead_Locked();
256 void AudioRendererImpl::Initialize(DemuxerStream
* stream
,
257 const PipelineStatusCB
& init_cb
,
258 const StatisticsCB
& statistics_cb
,
259 const BufferingStateCB
& buffering_state_cb
,
260 const base::Closure
& ended_cb
,
261 const PipelineStatusCB
& error_cb
) {
262 DVLOG(1) << __FUNCTION__
;
263 DCHECK(task_runner_
->BelongsToCurrentThread());
265 DCHECK_EQ(stream
->type(), DemuxerStream::AUDIO
);
266 DCHECK(!init_cb
.is_null());
267 DCHECK(!statistics_cb
.is_null());
268 DCHECK(!buffering_state_cb
.is_null());
269 DCHECK(!ended_cb
.is_null());
270 DCHECK(!error_cb
.is_null());
271 DCHECK_EQ(kUninitialized
, state_
);
274 state_
= kInitializing
;
276 // Always post |init_cb_| because |this| could be destroyed if initialization
278 init_cb_
= BindToCurrentLoop(init_cb
);
280 buffering_state_cb_
= buffering_state_cb
;
281 ended_cb_
= ended_cb
;
282 error_cb_
= error_cb
;
284 expecting_config_changes_
= stream
->SupportsConfigChanges();
285 if (!expecting_config_changes_
) {
286 // The actual buffer size is controlled via the size of the AudioBus
287 // provided to Render(), so just choose something reasonable here for looks.
288 int buffer_size
= stream
->audio_decoder_config().samples_per_second() / 100;
289 audio_parameters_
.Reset(
290 AudioParameters::AUDIO_PCM_LOW_LATENCY
,
291 stream
->audio_decoder_config().channel_layout(),
292 ChannelLayoutToChannelCount(
293 stream
->audio_decoder_config().channel_layout()),
294 stream
->audio_decoder_config().samples_per_second(),
295 stream
->audio_decoder_config().bits_per_channel(),
297 buffer_converter_
.reset();
299 // TODO(rileya): Support hardware config changes
300 const AudioParameters
& hw_params
= hardware_config_
.GetOutputConfig();
301 audio_parameters_
.Reset(
303 // Always use the source's channel layout and channel count to avoid
304 // premature downmixing (http://crbug.com/379288), platform specific
305 // issues around channel layouts (http://crbug.com/266674), and
306 // unnecessary upmixing overhead.
307 stream
->audio_decoder_config().channel_layout(),
308 ChannelLayoutToChannelCount(
309 stream
->audio_decoder_config().channel_layout()),
310 hw_params
.sample_rate(),
311 hw_params
.bits_per_sample(),
312 hardware_config_
.GetHighLatencyBufferSize());
316 new AudioClock(base::TimeDelta(), audio_parameters_
.sample_rate()));
318 audio_buffer_stream_
->Initialize(
319 stream
, statistics_cb
,
320 base::Bind(&AudioRendererImpl::OnAudioBufferStreamInitialized
,
321 weak_factory_
.GetWeakPtr()));
324 void AudioRendererImpl::OnAudioBufferStreamInitialized(bool success
) {
325 DVLOG(1) << __FUNCTION__
<< ": " << success
;
326 DCHECK(task_runner_
->BelongsToCurrentThread());
328 base::AutoLock
auto_lock(lock_
);
331 state_
= kUninitialized
;
332 base::ResetAndReturn(&init_cb_
).Run(DECODER_ERROR_NOT_SUPPORTED
);
336 if (!audio_parameters_
.IsValid()) {
337 DVLOG(1) << __FUNCTION__
<< ": Invalid audio parameters: "
338 << audio_parameters_
.AsHumanReadableString();
339 ChangeState_Locked(kUninitialized
);
340 base::ResetAndReturn(&init_cb_
).Run(PIPELINE_ERROR_INITIALIZATION_FAILED
);
344 if (expecting_config_changes_
)
345 buffer_converter_
.reset(new AudioBufferConverter(audio_parameters_
));
346 splicer_
.reset(new AudioSplicer(audio_parameters_
.sample_rate()));
348 // We're all good! Continue initializing the rest of the audio renderer
349 // based on the decoder format.
350 algorithm_
.reset(new AudioRendererAlgorithm());
351 algorithm_
->Initialize(audio_parameters_
);
353 ChangeState_Locked(kFlushed
);
355 HistogramRendererEvent(INITIALIZED
);
358 base::AutoUnlock
auto_unlock(lock_
);
359 sink_
->Initialize(audio_parameters_
, this);
362 // Some sinks play on start...
366 DCHECK(!sink_playing_
);
367 base::ResetAndReturn(&init_cb_
).Run(PIPELINE_OK
);
370 void AudioRendererImpl::SetVolume(float volume
) {
371 DCHECK(task_runner_
->BelongsToCurrentThread());
373 sink_
->SetVolume(volume
);
376 void AudioRendererImpl::DecodedAudioReady(
377 AudioBufferStream::Status status
,
378 const scoped_refptr
<AudioBuffer
>& buffer
) {
379 DVLOG(2) << __FUNCTION__
<< "(" << status
<< ")";
380 DCHECK(task_runner_
->BelongsToCurrentThread());
382 base::AutoLock
auto_lock(lock_
);
383 DCHECK(state_
!= kUninitialized
);
385 CHECK(pending_read_
);
386 pending_read_
= false;
388 if (status
== AudioBufferStream::ABORTED
||
389 status
== AudioBufferStream::DEMUXER_READ_ABORTED
) {
390 HandleAbortedReadOrDecodeError(false);
394 if (status
== AudioBufferStream::DECODE_ERROR
) {
395 HandleAbortedReadOrDecodeError(true);
399 DCHECK_EQ(status
, AudioBufferStream::OK
);
400 DCHECK(buffer
.get());
402 if (state_
== kFlushing
) {
403 ChangeState_Locked(kFlushed
);
408 if (expecting_config_changes_
) {
409 DCHECK(buffer_converter_
);
410 buffer_converter_
->AddInput(buffer
);
411 while (buffer_converter_
->HasNextBuffer()) {
412 if (!splicer_
->AddInput(buffer_converter_
->GetNextBuffer())) {
413 HandleAbortedReadOrDecodeError(true);
418 if (!splicer_
->AddInput(buffer
)) {
419 HandleAbortedReadOrDecodeError(true);
424 if (!splicer_
->HasNextBuffer()) {
425 AttemptRead_Locked();
429 bool need_another_buffer
= false;
430 while (splicer_
->HasNextBuffer())
431 need_another_buffer
= HandleSplicerBuffer_Locked(splicer_
->GetNextBuffer());
433 if (!need_another_buffer
&& !CanRead_Locked())
436 AttemptRead_Locked();
439 bool AudioRendererImpl::HandleSplicerBuffer_Locked(
440 const scoped_refptr
<AudioBuffer
>& buffer
) {
441 lock_
.AssertAcquired();
442 if (buffer
->end_of_stream()) {
443 received_end_of_stream_
= true;
445 if (state_
== kPlaying
) {
446 if (IsBeforeStartTime(buffer
))
449 // Trim off any additional time before the start timestamp.
450 const base::TimeDelta trim_time
= start_timestamp_
- buffer
->timestamp();
451 if (trim_time
> base::TimeDelta()) {
452 buffer
->TrimStart(buffer
->frame_count() *
453 (static_cast<double>(trim_time
.InMicroseconds()) /
454 buffer
->duration().InMicroseconds()));
456 // If the entire buffer was trimmed, request a new one.
457 if (!buffer
->frame_count())
461 if (state_
!= kUninitialized
)
462 algorithm_
->EnqueueBuffer(buffer
);
473 DCHECK(!pending_read_
);
477 if (buffer
->end_of_stream() || algorithm_
->IsQueueFull()) {
478 if (buffering_state_
== BUFFERING_HAVE_NOTHING
)
479 SetBufferingState_Locked(BUFFERING_HAVE_ENOUGH
);
487 void AudioRendererImpl::AttemptRead() {
488 base::AutoLock
auto_lock(lock_
);
489 AttemptRead_Locked();
492 void AudioRendererImpl::AttemptRead_Locked() {
493 DCHECK(task_runner_
->BelongsToCurrentThread());
494 lock_
.AssertAcquired();
496 if (!CanRead_Locked())
499 pending_read_
= true;
500 audio_buffer_stream_
->Read(base::Bind(&AudioRendererImpl::DecodedAudioReady
,
501 weak_factory_
.GetWeakPtr()));
504 bool AudioRendererImpl::CanRead_Locked() {
505 lock_
.AssertAcquired();
518 return !pending_read_
&& !received_end_of_stream_
&&
519 !algorithm_
->IsQueueFull();
522 void AudioRendererImpl::SetPlaybackRate(float playback_rate
) {
523 DVLOG(1) << __FUNCTION__
<< "(" << playback_rate
<< ")";
524 DCHECK(task_runner_
->BelongsToCurrentThread());
525 DCHECK_GE(playback_rate
, 0);
528 base::AutoLock
auto_lock(lock_
);
530 // We have two cases here:
531 // Play: current_playback_rate == 0 && playback_rate != 0
532 // Pause: current_playback_rate != 0 && playback_rate == 0
533 float current_playback_rate
= playback_rate_
;
534 playback_rate_
= playback_rate
;
539 if (current_playback_rate
== 0 && playback_rate
!= 0) {
540 StartRendering_Locked();
544 if (current_playback_rate
!= 0 && playback_rate
== 0) {
545 StopRendering_Locked();
550 bool AudioRendererImpl::IsBeforeStartTime(
551 const scoped_refptr
<AudioBuffer
>& buffer
) {
552 DCHECK_EQ(state_
, kPlaying
);
553 return buffer
.get() && !buffer
->end_of_stream() &&
554 (buffer
->timestamp() + buffer
->duration()) < start_timestamp_
;
557 int AudioRendererImpl::Render(AudioBus
* audio_bus
,
558 int audio_delay_milliseconds
) {
559 const int requested_frames
= audio_bus
->frames();
560 base::TimeDelta playback_delay
= base::TimeDelta::FromMilliseconds(
561 audio_delay_milliseconds
);
562 const int delay_frames
= static_cast<int>(playback_delay
.InSecondsF() *
563 audio_parameters_
.sample_rate());
564 int frames_written
= 0;
566 base::AutoLock
auto_lock(lock_
);
567 last_render_ticks_
= base::TimeTicks::Now();
569 // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
571 audio_clock_
->WroteAudio(
572 0, requested_frames
, delay_frames
, playback_rate_
);
576 if (playback_rate_
== 0) {
577 audio_clock_
->WroteAudio(
578 0, requested_frames
, delay_frames
, playback_rate_
);
582 // Mute audio by returning 0 when not playing.
583 if (state_
!= kPlaying
) {
584 audio_clock_
->WroteAudio(
585 0, requested_frames
, delay_frames
, playback_rate_
);
589 // We use the following conditions to determine end of playback:
590 // 1) Algorithm can not fill the audio callback buffer
591 // 2) We received an end of stream buffer
592 // 3) We haven't already signalled that we've ended
593 // 4) We've played all known audio data sent to hardware
595 // We use the following conditions to determine underflow:
596 // 1) Algorithm can not fill the audio callback buffer
597 // 2) We have NOT received an end of stream buffer
598 // 3) We are in the kPlaying state
600 // Otherwise the buffer has data we can send to the device.
601 if (algorithm_
->frames_buffered() > 0) {
603 algorithm_
->FillBuffer(audio_bus
, requested_frames
, playback_rate_
);
606 // Per the TimeSource API the media time should always increase even after
607 // we've rendered all known audio data. Doing so simplifies scenarios where
608 // we have other sources of media data that need to be scheduled after audio
611 // That being said, we don't want to advance time when underflowed as we
612 // know more decoded frames will eventually arrive. If we did, we would
613 // throw things out of sync when said decoded frames arrive.
614 int frames_after_end_of_stream
= 0;
615 if (frames_written
== 0) {
616 if (received_end_of_stream_
) {
617 if (ended_timestamp_
== kInfiniteDuration())
618 ended_timestamp_
= audio_clock_
->back_timestamp();
619 frames_after_end_of_stream
= requested_frames
;
620 } else if (state_
== kPlaying
&&
621 buffering_state_
!= BUFFERING_HAVE_NOTHING
) {
622 algorithm_
->IncreaseQueueCapacity();
623 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING
);
627 audio_clock_
->WroteAudio(frames_written
+ frames_after_end_of_stream
,
632 if (CanRead_Locked()) {
633 task_runner_
->PostTask(FROM_HERE
,
634 base::Bind(&AudioRendererImpl::AttemptRead
,
635 weak_factory_
.GetWeakPtr()));
638 if (audio_clock_
->front_timestamp() >= ended_timestamp_
&&
639 !rendered_end_of_stream_
) {
640 rendered_end_of_stream_
= true;
641 task_runner_
->PostTask(FROM_HERE
, ended_cb_
);
645 DCHECK_LE(frames_written
, requested_frames
);
646 return frames_written
;
649 void AudioRendererImpl::OnRenderError() {
650 // UMA data tells us this happens ~0.01% of the time. Trigger an error instead
651 // of trying to gracefully fall back to a fake sink. It's very likely
652 // OnRenderError() should be removed and the audio stack handle errors without
653 // notifying clients. See http://crbug.com/234708 for details.
654 HistogramRendererEvent(RENDER_ERROR
);
655 // Post to |task_runner_| as this is called on the audio callback thread.
656 task_runner_
->PostTask(FROM_HERE
,
657 base::Bind(error_cb_
, PIPELINE_ERROR_DECODE
));
660 void AudioRendererImpl::HandleAbortedReadOrDecodeError(bool is_decode_error
) {
661 DCHECK(task_runner_
->BelongsToCurrentThread());
662 lock_
.AssertAcquired();
664 PipelineStatus status
= is_decode_error
? PIPELINE_ERROR_DECODE
: PIPELINE_OK
;
671 ChangeState_Locked(kFlushed
);
672 if (status
== PIPELINE_OK
) {
677 error_cb_
.Run(status
);
678 base::ResetAndReturn(&flush_cb_
).Run();
683 if (status
!= PIPELINE_OK
)
684 error_cb_
.Run(status
);
689 void AudioRendererImpl::ChangeState_Locked(State new_state
) {
690 DVLOG(1) << __FUNCTION__
<< " : " << state_
<< " -> " << new_state
;
691 lock_
.AssertAcquired();
695 void AudioRendererImpl::OnNewSpliceBuffer(base::TimeDelta splice_timestamp
) {
696 DCHECK(task_runner_
->BelongsToCurrentThread());
697 splicer_
->SetSpliceTimestamp(splice_timestamp
);
700 void AudioRendererImpl::OnConfigChange() {
701 DCHECK(task_runner_
->BelongsToCurrentThread());
702 DCHECK(expecting_config_changes_
);
703 buffer_converter_
->ResetTimestampState();
704 // Drain flushed buffers from the converter so the AudioSplicer receives all
705 // data ahead of any OnNewSpliceBuffer() calls. Since discontinuities should
706 // only appear after config changes, AddInput() should never fail here.
707 while (buffer_converter_
->HasNextBuffer())
708 CHECK(splicer_
->AddInput(buffer_converter_
->GetNextBuffer()));
711 void AudioRendererImpl::SetBufferingState_Locked(
712 BufferingState buffering_state
) {
713 DVLOG(1) << __FUNCTION__
<< " : " << buffering_state_
<< " -> "
715 DCHECK_NE(buffering_state_
, buffering_state
);
716 lock_
.AssertAcquired();
717 buffering_state_
= buffering_state
;
719 task_runner_
->PostTask(FROM_HERE
,
720 base::Bind(buffering_state_cb_
, buffering_state_
));