1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/filters/audio_renderer_impl.h"
11 #include "base/bind.h"
12 #include "base/callback.h"
13 #include "base/callback_helpers.h"
14 #include "base/logging.h"
15 #include "base/metrics/histogram.h"
16 #include "base/single_thread_task_runner.h"
17 #include "media/base/audio_buffer.h"
18 #include "media/base/audio_buffer_converter.h"
19 #include "media/base/audio_hardware_config.h"
20 #include "media/base/audio_splicer.h"
21 #include "media/base/bind_to_current_loop.h"
22 #include "media/base/demuxer_stream.h"
23 #include "media/filters/audio_clock.h"
24 #include "media/filters/decrypting_demuxer_stream.h"
30 enum AudioRendererEvent
{
33 RENDER_EVENT_MAX
= RENDER_ERROR
,
36 void HistogramRendererEvent(AudioRendererEvent event
) {
37 UMA_HISTOGRAM_ENUMERATION(
38 "Media.AudioRendererEvents", event
, RENDER_EVENT_MAX
+ 1);
43 AudioRendererImpl::AudioRendererImpl(
44 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
,
45 media::AudioRendererSink
* sink
,
46 ScopedVector
<AudioDecoder
> decoders
,
47 const SetDecryptorReadyCB
& set_decryptor_ready_cb
,
48 const AudioHardwareConfig
& hardware_config
,
49 const scoped_refptr
<MediaLog
>& media_log
)
50 : task_runner_(task_runner
),
51 expecting_config_changes_(false),
53 audio_buffer_stream_(new AudioBufferStream(task_runner
,
55 set_decryptor_ready_cb
,
57 hardware_config_(hardware_config
),
59 state_(kUninitialized
),
60 buffering_state_(BUFFERING_HAVE_NOTHING
),
64 received_end_of_stream_(false),
65 rendered_end_of_stream_(false),
67 audio_buffer_stream_
->set_splice_observer(base::Bind(
68 &AudioRendererImpl::OnNewSpliceBuffer
, weak_factory_
.GetWeakPtr()));
69 audio_buffer_stream_
->set_config_change_observer(base::Bind(
70 &AudioRendererImpl::OnConfigChange
, weak_factory_
.GetWeakPtr()));
73 AudioRendererImpl::~AudioRendererImpl() {
74 DVLOG(1) << __FUNCTION__
;
75 DCHECK(task_runner_
->BelongsToCurrentThread());
77 // If Render() is in progress, this call will wait for Render() to finish.
78 // After this call, the |sink_| will not call back into |this| anymore.
81 if (!init_cb_
.is_null())
82 base::ResetAndReturn(&init_cb_
).Run(PIPELINE_ERROR_ABORT
);
85 void AudioRendererImpl::StartTicking() {
86 DVLOG(1) << __FUNCTION__
;
87 DCHECK(task_runner_
->BelongsToCurrentThread());
91 base::AutoLock
auto_lock(lock_
);
92 // Wait for an eventual call to SetPlaybackRate() to start rendering.
93 if (playback_rate_
== 0) {
94 DCHECK(!sink_playing_
);
98 StartRendering_Locked();
101 void AudioRendererImpl::StartRendering_Locked() {
102 DVLOG(1) << __FUNCTION__
;
103 DCHECK(task_runner_
->BelongsToCurrentThread());
104 DCHECK_EQ(state_
, kPlaying
);
105 DCHECK(!sink_playing_
);
106 DCHECK_NE(playback_rate_
, 0);
107 lock_
.AssertAcquired();
109 sink_playing_
= true;
111 base::AutoUnlock
auto_unlock(lock_
);
115 void AudioRendererImpl::StopTicking() {
116 DVLOG(1) << __FUNCTION__
;
117 DCHECK(task_runner_
->BelongsToCurrentThread());
121 base::AutoLock
auto_lock(lock_
);
122 // Rendering should have already been stopped with a zero playback rate.
123 if (playback_rate_
== 0) {
124 DCHECK(!sink_playing_
);
128 StopRendering_Locked();
131 void AudioRendererImpl::StopRendering_Locked() {
132 DCHECK(task_runner_
->BelongsToCurrentThread());
133 DCHECK_EQ(state_
, kPlaying
);
134 DCHECK(sink_playing_
);
135 lock_
.AssertAcquired();
137 sink_playing_
= false;
139 base::AutoUnlock
auto_unlock(lock_
);
143 void AudioRendererImpl::SetMediaTime(base::TimeDelta time
) {
144 DVLOG(1) << __FUNCTION__
<< "(" << time
.InMicroseconds() << ")";
145 DCHECK(task_runner_
->BelongsToCurrentThread());
147 base::AutoLock
auto_lock(lock_
);
149 DCHECK_EQ(state_
, kFlushed
);
151 start_timestamp_
= time
;
152 ended_timestamp_
= kInfiniteDuration();
153 last_render_ticks_
= base::TimeTicks();
154 audio_clock_
.reset(new AudioClock(time
, audio_parameters_
.sample_rate()));
157 base::TimeDelta
AudioRendererImpl::CurrentMediaTime() {
158 DVLOG(2) << __FUNCTION__
;
160 // In practice the Render() method is called with a high enough frequency
161 // that returning only the front timestamp is good enough and also prevents
162 // returning values that go backwards in time.
163 base::AutoLock
auto_lock(lock_
);
164 return audio_clock_
->front_timestamp();
167 base::TimeDelta
AudioRendererImpl::CurrentMediaTimeForSyncingVideo() {
168 DVLOG(2) << __FUNCTION__
;
170 base::AutoLock
auto_lock(lock_
);
171 if (last_render_ticks_
.is_null())
172 return audio_clock_
->front_timestamp();
174 return audio_clock_
->TimestampSinceWriting(base::TimeTicks::Now() -
178 TimeSource
* AudioRendererImpl::GetTimeSource() {
182 void AudioRendererImpl::Flush(const base::Closure
& callback
) {
183 DVLOG(1) << __FUNCTION__
;
184 DCHECK(task_runner_
->BelongsToCurrentThread());
186 base::AutoLock
auto_lock(lock_
);
187 DCHECK_EQ(state_
, kPlaying
);
188 DCHECK(flush_cb_
.is_null());
190 flush_cb_
= callback
;
191 ChangeState_Locked(kFlushing
);
196 ChangeState_Locked(kFlushed
);
200 void AudioRendererImpl::DoFlush_Locked() {
201 DCHECK(task_runner_
->BelongsToCurrentThread());
202 lock_
.AssertAcquired();
204 DCHECK(!pending_read_
);
205 DCHECK_EQ(state_
, kFlushed
);
207 audio_buffer_stream_
->Reset(base::Bind(&AudioRendererImpl::ResetDecoderDone
,
208 weak_factory_
.GetWeakPtr()));
211 void AudioRendererImpl::ResetDecoderDone() {
212 DCHECK(task_runner_
->BelongsToCurrentThread());
214 base::AutoLock
auto_lock(lock_
);
216 DCHECK_EQ(state_
, kFlushed
);
217 DCHECK(!flush_cb_
.is_null());
219 received_end_of_stream_
= false;
220 rendered_end_of_stream_
= false;
222 // Flush() may have been called while underflowed/not fully buffered.
223 if (buffering_state_
!= BUFFERING_HAVE_NOTHING
)
224 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING
);
227 if (buffer_converter_
)
228 buffer_converter_
->Reset();
229 algorithm_
->FlushBuffers();
232 // Changes in buffering state are always posted. Flush callback must only be
233 // run after buffering state has been set back to nothing.
234 task_runner_
->PostTask(FROM_HERE
, base::ResetAndReturn(&flush_cb_
));
237 void AudioRendererImpl::StartPlaying() {
238 DVLOG(1) << __FUNCTION__
;
239 DCHECK(task_runner_
->BelongsToCurrentThread());
241 base::AutoLock
auto_lock(lock_
);
242 DCHECK(!sink_playing_
);
243 DCHECK_EQ(state_
, kFlushed
);
244 DCHECK_EQ(buffering_state_
, BUFFERING_HAVE_NOTHING
);
245 DCHECK(!pending_read_
) << "Pending read must complete before seeking";
247 ChangeState_Locked(kPlaying
);
248 AttemptRead_Locked();
251 void AudioRendererImpl::Initialize(DemuxerStream
* stream
,
252 const PipelineStatusCB
& init_cb
,
253 const StatisticsCB
& statistics_cb
,
254 const BufferingStateCB
& buffering_state_cb
,
255 const base::Closure
& ended_cb
,
256 const PipelineStatusCB
& error_cb
) {
257 DCHECK(task_runner_
->BelongsToCurrentThread());
259 DCHECK_EQ(stream
->type(), DemuxerStream::AUDIO
);
260 DCHECK(!init_cb
.is_null());
261 DCHECK(!statistics_cb
.is_null());
262 DCHECK(!buffering_state_cb
.is_null());
263 DCHECK(!ended_cb
.is_null());
264 DCHECK(!error_cb
.is_null());
265 DCHECK_EQ(kUninitialized
, state_
);
268 state_
= kInitializing
;
270 // Always post |init_cb_| because |this| could be destroyed if initialization
272 init_cb_
= BindToCurrentLoop(init_cb
);
274 buffering_state_cb_
= buffering_state_cb
;
275 ended_cb_
= ended_cb
;
276 error_cb_
= error_cb
;
278 expecting_config_changes_
= stream
->SupportsConfigChanges();
279 if (!expecting_config_changes_
) {
280 // The actual buffer size is controlled via the size of the AudioBus
281 // provided to Render(), so just choose something reasonable here for looks.
282 int buffer_size
= stream
->audio_decoder_config().samples_per_second() / 100;
283 audio_parameters_
.Reset(
284 AudioParameters::AUDIO_PCM_LOW_LATENCY
,
285 stream
->audio_decoder_config().channel_layout(),
286 ChannelLayoutToChannelCount(
287 stream
->audio_decoder_config().channel_layout()),
288 stream
->audio_decoder_config().samples_per_second(),
289 stream
->audio_decoder_config().bits_per_channel(),
291 buffer_converter_
.reset();
293 // TODO(rileya): Support hardware config changes
294 const AudioParameters
& hw_params
= hardware_config_
.GetOutputConfig();
295 audio_parameters_
.Reset(
297 // Always use the source's channel layout and channel count to avoid
298 // premature downmixing (http://crbug.com/379288), platform specific
299 // issues around channel layouts (http://crbug.com/266674), and
300 // unnecessary upmixing overhead.
301 stream
->audio_decoder_config().channel_layout(),
302 ChannelLayoutToChannelCount(
303 stream
->audio_decoder_config().channel_layout()),
304 hw_params
.sample_rate(),
305 hw_params
.bits_per_sample(),
306 hardware_config_
.GetHighLatencyBufferSize());
310 new AudioClock(base::TimeDelta(), audio_parameters_
.sample_rate()));
312 audio_buffer_stream_
->Initialize(
316 base::Bind(&AudioRendererImpl::OnAudioBufferStreamInitialized
,
317 weak_factory_
.GetWeakPtr()));
320 void AudioRendererImpl::OnAudioBufferStreamInitialized(bool success
) {
321 DCHECK(task_runner_
->BelongsToCurrentThread());
323 base::AutoLock
auto_lock(lock_
);
326 state_
= kUninitialized
;
327 base::ResetAndReturn(&init_cb_
).Run(DECODER_ERROR_NOT_SUPPORTED
);
331 if (!audio_parameters_
.IsValid()) {
332 ChangeState_Locked(kUninitialized
);
333 base::ResetAndReturn(&init_cb_
).Run(PIPELINE_ERROR_INITIALIZATION_FAILED
);
337 if (expecting_config_changes_
)
338 buffer_converter_
.reset(new AudioBufferConverter(audio_parameters_
));
339 splicer_
.reset(new AudioSplicer(audio_parameters_
.sample_rate()));
341 // We're all good! Continue initializing the rest of the audio renderer
342 // based on the decoder format.
343 algorithm_
.reset(new AudioRendererAlgorithm());
344 algorithm_
->Initialize(audio_parameters_
);
346 ChangeState_Locked(kFlushed
);
348 HistogramRendererEvent(INITIALIZED
);
351 base::AutoUnlock
auto_unlock(lock_
);
352 sink_
->Initialize(audio_parameters_
, this);
355 // Some sinks play on start...
359 DCHECK(!sink_playing_
);
360 base::ResetAndReturn(&init_cb_
).Run(PIPELINE_OK
);
363 void AudioRendererImpl::SetVolume(float volume
) {
364 DCHECK(task_runner_
->BelongsToCurrentThread());
366 sink_
->SetVolume(volume
);
369 void AudioRendererImpl::DecodedAudioReady(
370 AudioBufferStream::Status status
,
371 const scoped_refptr
<AudioBuffer
>& buffer
) {
372 DVLOG(2) << __FUNCTION__
<< "(" << status
<< ")";
373 DCHECK(task_runner_
->BelongsToCurrentThread());
375 base::AutoLock
auto_lock(lock_
);
376 DCHECK(state_
!= kUninitialized
);
378 CHECK(pending_read_
);
379 pending_read_
= false;
381 if (status
== AudioBufferStream::ABORTED
||
382 status
== AudioBufferStream::DEMUXER_READ_ABORTED
) {
383 HandleAbortedReadOrDecodeError(false);
387 if (status
== AudioBufferStream::DECODE_ERROR
) {
388 HandleAbortedReadOrDecodeError(true);
392 DCHECK_EQ(status
, AudioBufferStream::OK
);
393 DCHECK(buffer
.get());
395 if (state_
== kFlushing
) {
396 ChangeState_Locked(kFlushed
);
401 if (expecting_config_changes_
) {
402 DCHECK(buffer_converter_
);
403 buffer_converter_
->AddInput(buffer
);
404 while (buffer_converter_
->HasNextBuffer()) {
405 if (!splicer_
->AddInput(buffer_converter_
->GetNextBuffer())) {
406 HandleAbortedReadOrDecodeError(true);
411 if (!splicer_
->AddInput(buffer
)) {
412 HandleAbortedReadOrDecodeError(true);
417 if (!splicer_
->HasNextBuffer()) {
418 AttemptRead_Locked();
422 bool need_another_buffer
= false;
423 while (splicer_
->HasNextBuffer())
424 need_another_buffer
= HandleSplicerBuffer_Locked(splicer_
->GetNextBuffer());
426 if (!need_another_buffer
&& !CanRead_Locked())
429 AttemptRead_Locked();
432 bool AudioRendererImpl::HandleSplicerBuffer_Locked(
433 const scoped_refptr
<AudioBuffer
>& buffer
) {
434 lock_
.AssertAcquired();
435 if (buffer
->end_of_stream()) {
436 received_end_of_stream_
= true;
438 if (state_
== kPlaying
) {
439 if (IsBeforeStartTime(buffer
))
442 // Trim off any additional time before the start timestamp.
443 const base::TimeDelta trim_time
= start_timestamp_
- buffer
->timestamp();
444 if (trim_time
> base::TimeDelta()) {
445 buffer
->TrimStart(buffer
->frame_count() *
446 (static_cast<double>(trim_time
.InMicroseconds()) /
447 buffer
->duration().InMicroseconds()));
449 // If the entire buffer was trimmed, request a new one.
450 if (!buffer
->frame_count())
454 if (state_
!= kUninitialized
)
455 algorithm_
->EnqueueBuffer(buffer
);
466 DCHECK(!pending_read_
);
470 if (buffer
->end_of_stream() || algorithm_
->IsQueueFull()) {
471 if (buffering_state_
== BUFFERING_HAVE_NOTHING
)
472 SetBufferingState_Locked(BUFFERING_HAVE_ENOUGH
);
480 void AudioRendererImpl::AttemptRead() {
481 base::AutoLock
auto_lock(lock_
);
482 AttemptRead_Locked();
485 void AudioRendererImpl::AttemptRead_Locked() {
486 DCHECK(task_runner_
->BelongsToCurrentThread());
487 lock_
.AssertAcquired();
489 if (!CanRead_Locked())
492 pending_read_
= true;
493 audio_buffer_stream_
->Read(base::Bind(&AudioRendererImpl::DecodedAudioReady
,
494 weak_factory_
.GetWeakPtr()));
497 bool AudioRendererImpl::CanRead_Locked() {
498 lock_
.AssertAcquired();
511 return !pending_read_
&& !received_end_of_stream_
&&
512 !algorithm_
->IsQueueFull();
515 void AudioRendererImpl::SetPlaybackRate(float playback_rate
) {
516 DVLOG(1) << __FUNCTION__
<< "(" << playback_rate
<< ")";
517 DCHECK(task_runner_
->BelongsToCurrentThread());
518 DCHECK_GE(playback_rate
, 0);
521 base::AutoLock
auto_lock(lock_
);
523 // We have two cases here:
524 // Play: current_playback_rate == 0 && playback_rate != 0
525 // Pause: current_playback_rate != 0 && playback_rate == 0
526 float current_playback_rate
= playback_rate_
;
527 playback_rate_
= playback_rate
;
532 if (current_playback_rate
== 0 && playback_rate
!= 0) {
533 StartRendering_Locked();
537 if (current_playback_rate
!= 0 && playback_rate
== 0) {
538 StopRendering_Locked();
543 bool AudioRendererImpl::IsBeforeStartTime(
544 const scoped_refptr
<AudioBuffer
>& buffer
) {
545 DCHECK_EQ(state_
, kPlaying
);
546 return buffer
.get() && !buffer
->end_of_stream() &&
547 (buffer
->timestamp() + buffer
->duration()) < start_timestamp_
;
550 int AudioRendererImpl::Render(AudioBus
* audio_bus
,
551 int audio_delay_milliseconds
) {
552 const int requested_frames
= audio_bus
->frames();
553 base::TimeDelta playback_delay
= base::TimeDelta::FromMilliseconds(
554 audio_delay_milliseconds
);
555 const int delay_frames
= static_cast<int>(playback_delay
.InSecondsF() *
556 audio_parameters_
.sample_rate());
557 int frames_written
= 0;
559 base::AutoLock
auto_lock(lock_
);
560 last_render_ticks_
= base::TimeTicks::Now();
562 // Ensure Stop() hasn't destroyed our |algorithm_| on the pipeline thread.
564 audio_clock_
->WroteAudio(
565 0, requested_frames
, delay_frames
, playback_rate_
);
569 if (playback_rate_
== 0) {
570 audio_clock_
->WroteAudio(
571 0, requested_frames
, delay_frames
, playback_rate_
);
575 // Mute audio by returning 0 when not playing.
576 if (state_
!= kPlaying
) {
577 audio_clock_
->WroteAudio(
578 0, requested_frames
, delay_frames
, playback_rate_
);
582 // We use the following conditions to determine end of playback:
583 // 1) Algorithm can not fill the audio callback buffer
584 // 2) We received an end of stream buffer
585 // 3) We haven't already signalled that we've ended
586 // 4) We've played all known audio data sent to hardware
588 // We use the following conditions to determine underflow:
589 // 1) Algorithm can not fill the audio callback buffer
590 // 2) We have NOT received an end of stream buffer
591 // 3) We are in the kPlaying state
593 // Otherwise the buffer has data we can send to the device.
594 if (algorithm_
->frames_buffered() > 0) {
596 algorithm_
->FillBuffer(audio_bus
, requested_frames
, playback_rate_
);
599 // Per the TimeSource API the media time should always increase even after
600 // we've rendered all known audio data. Doing so simplifies scenarios where
601 // we have other sources of media data that need to be scheduled after audio
604 // That being said, we don't want to advance time when underflowed as we
605 // know more decoded frames will eventually arrive. If we did, we would
606 // throw things out of sync when said decoded frames arrive.
607 int frames_after_end_of_stream
= 0;
608 if (frames_written
== 0) {
609 if (received_end_of_stream_
) {
610 if (ended_timestamp_
== kInfiniteDuration())
611 ended_timestamp_
= audio_clock_
->back_timestamp();
612 frames_after_end_of_stream
= requested_frames
;
613 } else if (state_
== kPlaying
&&
614 buffering_state_
!= BUFFERING_HAVE_NOTHING
) {
615 algorithm_
->IncreaseQueueCapacity();
616 SetBufferingState_Locked(BUFFERING_HAVE_NOTHING
);
620 audio_clock_
->WroteAudio(frames_written
+ frames_after_end_of_stream
,
625 if (CanRead_Locked()) {
626 task_runner_
->PostTask(FROM_HERE
,
627 base::Bind(&AudioRendererImpl::AttemptRead
,
628 weak_factory_
.GetWeakPtr()));
631 if (audio_clock_
->front_timestamp() >= ended_timestamp_
&&
632 !rendered_end_of_stream_
) {
633 rendered_end_of_stream_
= true;
634 task_runner_
->PostTask(FROM_HERE
, ended_cb_
);
638 DCHECK_LE(frames_written
, requested_frames
);
639 return frames_written
;
642 void AudioRendererImpl::OnRenderError() {
643 // UMA data tells us this happens ~0.01% of the time. Trigger an error instead
644 // of trying to gracefully fall back to a fake sink. It's very likely
645 // OnRenderError() should be removed and the audio stack handle errors without
646 // notifying clients. See http://crbug.com/234708 for details.
647 HistogramRendererEvent(RENDER_ERROR
);
648 // Post to |task_runner_| as this is called on the audio callback thread.
649 task_runner_
->PostTask(FROM_HERE
,
650 base::Bind(error_cb_
, PIPELINE_ERROR_DECODE
));
653 void AudioRendererImpl::HandleAbortedReadOrDecodeError(bool is_decode_error
) {
654 DCHECK(task_runner_
->BelongsToCurrentThread());
655 lock_
.AssertAcquired();
657 PipelineStatus status
= is_decode_error
? PIPELINE_ERROR_DECODE
: PIPELINE_OK
;
664 ChangeState_Locked(kFlushed
);
665 if (status
== PIPELINE_OK
) {
670 error_cb_
.Run(status
);
671 base::ResetAndReturn(&flush_cb_
).Run();
676 if (status
!= PIPELINE_OK
)
677 error_cb_
.Run(status
);
682 void AudioRendererImpl::ChangeState_Locked(State new_state
) {
683 DVLOG(1) << __FUNCTION__
<< " : " << state_
<< " -> " << new_state
;
684 lock_
.AssertAcquired();
688 void AudioRendererImpl::OnNewSpliceBuffer(base::TimeDelta splice_timestamp
) {
689 DCHECK(task_runner_
->BelongsToCurrentThread());
690 splicer_
->SetSpliceTimestamp(splice_timestamp
);
693 void AudioRendererImpl::OnConfigChange() {
694 DCHECK(task_runner_
->BelongsToCurrentThread());
695 DCHECK(expecting_config_changes_
);
696 buffer_converter_
->ResetTimestampState();
697 // Drain flushed buffers from the converter so the AudioSplicer receives all
698 // data ahead of any OnNewSpliceBuffer() calls. Since discontinuities should
699 // only appear after config changes, AddInput() should never fail here.
700 while (buffer_converter_
->HasNextBuffer())
701 CHECK(splicer_
->AddInput(buffer_converter_
->GetNextBuffer()));
704 void AudioRendererImpl::SetBufferingState_Locked(
705 BufferingState buffering_state
) {
706 DVLOG(1) << __FUNCTION__
<< " : " << buffering_state_
<< " -> "
708 DCHECK_NE(buffering_state_
, buffering_state
);
709 lock_
.AssertAcquired();
710 buffering_state_
= buffering_state
;
712 task_runner_
->PostTask(FROM_HERE
,
713 base::Bind(buffering_state_cb_
, buffering_state_
));