1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_renderer.h"
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/strings/string_util.h"
10 #include "base/strings/stringprintf.h"
11 #include "content/renderer/media/audio_device_factory.h"
12 #include "content/renderer/media/media_stream_dispatcher.h"
13 #include "content/renderer/media/webrtc_audio_device_impl.h"
14 #include "content/renderer/media/webrtc_logging.h"
15 #include "content/renderer/render_frame_impl.h"
16 #include "media/audio/audio_output_device.h"
17 #include "media/audio/audio_parameters.h"
18 #include "media/audio/sample_rates.h"
19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
24 #include "base/win/windows_version.h"
25 #include "media/audio/win/core_audio_util_win.h"
32 // We add a UMA histogram measuring the execution time of the Render() method
33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
34 // between each callback leads to one UMA update each 100ms.
35 const int kNumCallbacksBetweenRenderTimeHistograms
= 10;
37 // This is a simple wrapper class that's handed out to users of a shared
38 // WebRtcAudioRenderer instance. This class maintains the per-user 'playing'
39 // and 'started' states to avoid problems related to incorrect usage which
40 // might violate the implementation assumptions inside WebRtcAudioRenderer
41 // (see the play reference count).
42 class SharedAudioRenderer
: public MediaStreamAudioRenderer
{
44 // Callback definition for a callback that is called when when Play(), Pause()
45 // or SetVolume are called (whenever the internal |playing_state_| changes).
46 typedef base::Callback
<
47 void(const scoped_refptr
<webrtc::MediaStreamInterface
>&,
48 WebRtcAudioRenderer::PlayingState
*)> OnPlayStateChanged
;
51 const scoped_refptr
<MediaStreamAudioRenderer
>& delegate
,
52 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
,
53 const OnPlayStateChanged
& on_play_state_changed
)
54 : delegate_(delegate
), media_stream_(media_stream
), started_(false),
55 on_play_state_changed_(on_play_state_changed
) {
56 DCHECK(!on_play_state_changed_
.is_null());
57 DCHECK(media_stream_
.get());
61 ~SharedAudioRenderer() override
{
62 DCHECK(thread_checker_
.CalledOnValidThread());
63 DVLOG(1) << __FUNCTION__
;
67 void Start() override
{
68 DCHECK(thread_checker_
.CalledOnValidThread());
75 void Play() override
{
76 DCHECK(thread_checker_
.CalledOnValidThread());
78 if (playing_state_
.playing())
80 playing_state_
.set_playing(true);
81 on_play_state_changed_
.Run(media_stream_
, &playing_state_
);
84 void Pause() override
{
85 DCHECK(thread_checker_
.CalledOnValidThread());
87 if (!playing_state_
.playing())
89 playing_state_
.set_playing(false);
90 on_play_state_changed_
.Run(media_stream_
, &playing_state_
);
93 void Stop() override
{
94 DCHECK(thread_checker_
.CalledOnValidThread());
102 void SetVolume(float volume
) override
{
103 DCHECK(thread_checker_
.CalledOnValidThread());
104 DCHECK(volume
>= 0.0f
&& volume
<= 1.0f
);
105 playing_state_
.set_volume(volume
);
106 on_play_state_changed_
.Run(media_stream_
, &playing_state_
);
109 void SwitchOutputDevice(
110 const std::string
& device_id
,
111 const GURL
& security_origin
,
112 const media::SwitchOutputDeviceCB
& callback
) override
{
113 DCHECK(thread_checker_
.CalledOnValidThread());
114 DVLOG(1) << __FUNCTION__
115 << "(" << device_id
<< ", " << security_origin
<< ")";
116 delegate_
->SwitchOutputDevice(device_id
, security_origin
, callback
);
119 base::TimeDelta
GetCurrentRenderTime() const override
{
120 DCHECK(thread_checker_
.CalledOnValidThread());
121 return delegate_
->GetCurrentRenderTime();
124 bool IsLocalRenderer() const override
{
125 DCHECK(thread_checker_
.CalledOnValidThread());
126 return delegate_
->IsLocalRenderer();
130 base::ThreadChecker thread_checker_
;
131 const scoped_refptr
<MediaStreamAudioRenderer
> delegate_
;
132 const scoped_refptr
<webrtc::MediaStreamInterface
> media_stream_
;
134 WebRtcAudioRenderer::PlayingState playing_state_
;
135 OnPlayStateChanged on_play_state_changed_
;
138 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
139 // depending on whether or not an input element is currently open with
141 int GetCurrentDuckingFlag(int render_frame_id
) {
142 RenderFrameImpl
* const frame
=
143 RenderFrameImpl::FromRoutingID(render_frame_id
);
144 MediaStreamDispatcher
* const dispatcher
= frame
?
145 frame
->GetMediaStreamDispatcher() : NULL
;
146 if (dispatcher
&& dispatcher
->IsAudioDuckingActive()) {
147 return media::AudioParameters::DUCKING
;
150 return media::AudioParameters::NO_EFFECTS
;
155 int WebRtcAudioRenderer::GetOptimalBufferSize(int sample_rate
,
156 int hardware_buffer_size
) {
157 // Use native hardware buffer size as default. On Windows, we strive to open
158 // up using this native hardware buffer size to achieve best
159 // possible performance and to ensure that no FIFO is needed on the browser
160 // side to match the client request. That is why there is no #if case for
162 int frames_per_buffer
= hardware_buffer_size
;
164 #if defined(OS_LINUX) || defined(OS_MACOSX)
165 // On Linux and MacOS, the low level IO implementations on the browser side
166 // supports all buffer size the clients want. We use the native peer
167 // connection buffer size (10ms) to achieve best possible performance.
168 frames_per_buffer
= sample_rate
/ 100;
169 #elif defined(OS_ANDROID)
170 // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
171 // cases. Might not be possible to come up with the perfect solution using
172 // the render side only.
173 int frames_per_10ms
= sample_rate
/ 100;
174 if (frames_per_buffer
< 2 * frames_per_10ms
) {
175 // Examples of low-latency frame sizes and the resulting |buffer_size|:
176 // Nexus 7 : 240 audio frames => 2*480 = 960
177 // Nexus 10 : 256 => 2*441 = 882
178 // Galaxy Nexus: 144 => 2*441 = 882
179 frames_per_buffer
= 2 * frames_per_10ms
;
180 DVLOG(1) << "Low-latency output detected on Android";
184 DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer
;
185 return frames_per_buffer
;
188 WebRtcAudioRenderer::WebRtcAudioRenderer(
189 const scoped_refptr
<base::SingleThreadTaskRunner
>& signaling_thread
,
190 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
,
191 int source_render_frame_id
,
194 int frames_per_buffer
)
195 : state_(UNINITIALIZED
),
196 source_render_frame_id_(source_render_frame_id
),
197 session_id_(session_id
),
198 signaling_thread_(signaling_thread
),
199 media_stream_(media_stream
),
203 audio_delay_milliseconds_(0),
204 fifo_delay_milliseconds_(0),
205 sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY
,
206 media::CHANNEL_LAYOUT_STEREO
, sample_rate
, 16,
208 GetCurrentDuckingFlag(source_render_frame_id
)),
209 render_callback_count_(0) {
210 WebRtcLogMessage(base::StringPrintf(
211 "WAR::WAR. source_render_frame_id=%d"
212 ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
213 source_render_frame_id
, session_id
, sample_rate
, frames_per_buffer
,
214 sink_params_
.effects()));
217 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
218 DCHECK(thread_checker_
.CalledOnValidThread());
219 DCHECK_EQ(state_
, UNINITIALIZED
);
222 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource
* source
) {
223 DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
224 DCHECK(thread_checker_
.CalledOnValidThread());
225 base::AutoLock
auto_lock(lock_
);
226 DCHECK_EQ(state_
, UNINITIALIZED
);
228 DCHECK(!sink_
.get());
231 // WebRTC does not yet support higher rates than 96000 on the client side
232 // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
233 // we change the rate to 48000 instead. The consequence is that the native
234 // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
235 // which will then be resampled by the audio converted on the browser side
236 // to match the native audio layer.
237 int sample_rate
= sink_params_
.sample_rate();
238 DVLOG(1) << "Audio output hardware sample rate: " << sample_rate
;
239 if (sample_rate
== 192000) {
240 DVLOG(1) << "Resampling from 48000 to 192000 is required";
243 media::AudioSampleRate asr
;
244 if (media::ToAudioSampleRate(sample_rate
, &asr
)) {
245 UMA_HISTOGRAM_ENUMERATION(
246 "WebRTC.AudioOutputSampleRate", asr
, media::kAudioSampleRateMax
+ 1);
248 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
252 // Set up audio parameters for the source, i.e., the WebRTC client.
254 // The WebRTC client only supports multiples of 10ms as buffer size where
255 // 10ms is preferred for lowest possible delay.
256 media::AudioParameters source_params
;
257 const int frames_per_10ms
= (sample_rate
/ 100);
258 DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms
;
260 source_params
.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY
,
261 sink_params_
.channel_layout(), sink_params_
.channels(),
262 sample_rate
, 16, frames_per_10ms
);
264 const int frames_per_buffer
=
265 GetOptimalBufferSize(sample_rate
, sink_params_
.frames_per_buffer());
267 sink_params_
.Reset(sink_params_
.format(), sink_params_
.channel_layout(),
268 sink_params_
.channels(), sample_rate
, 16,
271 // Create a FIFO if re-buffering is required to match the source input with
272 // the sink request. The source acts as provider here and the sink as
274 fifo_delay_milliseconds_
= 0;
275 if (source_params
.frames_per_buffer() != sink_params_
.frames_per_buffer()) {
276 DVLOG(1) << "Rebuffering from " << source_params
.frames_per_buffer()
277 << " to " << sink_params_
.frames_per_buffer();
278 audio_fifo_
.reset(new media::AudioPullFifo(
279 source_params
.channels(),
280 source_params
.frames_per_buffer(),
282 &WebRtcAudioRenderer::SourceCallback
,
283 base::Unretained(this))));
285 if (sink_params_
.frames_per_buffer() > source_params
.frames_per_buffer()) {
286 int frame_duration_milliseconds
= base::Time::kMillisecondsPerSecond
/
287 static_cast<double>(source_params
.sample_rate());
288 fifo_delay_milliseconds_
= (sink_params_
.frames_per_buffer() -
289 source_params
.frames_per_buffer()) * frame_duration_milliseconds
;
295 // Configure the audio rendering client and start rendering.
296 sink_
= AudioDeviceFactory::NewOutputDevice(source_render_frame_id_
);
298 DCHECK_GE(session_id_
, 0);
299 sink_
->InitializeWithSessionId(sink_params_
, this, session_id_
);
303 // User must call Play() before any audio can be heard.
309 scoped_refptr
<MediaStreamAudioRenderer
>
310 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
311 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
) {
312 content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed
=
313 base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged
, this);
314 return new SharedAudioRenderer(this, media_stream
, on_play_state_changed
);
317 bool WebRtcAudioRenderer::IsStarted() const {
318 DCHECK(thread_checker_
.CalledOnValidThread());
319 return start_ref_count_
!= 0;
322 void WebRtcAudioRenderer::Start() {
323 DVLOG(1) << "WebRtcAudioRenderer::Start()";
324 DCHECK(thread_checker_
.CalledOnValidThread());
328 void WebRtcAudioRenderer::Play() {
329 DVLOG(1) << "WebRtcAudioRenderer::Play()";
330 DCHECK(thread_checker_
.CalledOnValidThread());
332 if (playing_state_
.playing())
335 playing_state_
.set_playing(true);
336 render_callback_count_
= 0;
338 OnPlayStateChanged(media_stream_
, &playing_state_
);
341 void WebRtcAudioRenderer::EnterPlayState() {
342 DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
343 DCHECK(thread_checker_
.CalledOnValidThread());
344 DCHECK_GT(start_ref_count_
, 0) << "Did you forget to call Start()?";
345 base::AutoLock
auto_lock(lock_
);
346 if (state_
== UNINITIALIZED
)
349 DCHECK(play_ref_count_
== 0 || state_
== PLAYING
);
352 if (state_
!= PLAYING
) {
356 audio_delay_milliseconds_
= 0;
357 audio_fifo_
->Clear();
362 void WebRtcAudioRenderer::Pause() {
363 DVLOG(1) << "WebRtcAudioRenderer::Pause()";
364 DCHECK(thread_checker_
.CalledOnValidThread());
365 if (!playing_state_
.playing())
368 playing_state_
.set_playing(false);
370 OnPlayStateChanged(media_stream_
, &playing_state_
);
373 void WebRtcAudioRenderer::EnterPauseState() {
374 DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
375 DCHECK(thread_checker_
.CalledOnValidThread());
376 DCHECK_GT(start_ref_count_
, 0) << "Did you forget to call Start()?";
377 base::AutoLock
auto_lock(lock_
);
378 if (state_
== UNINITIALIZED
)
381 DCHECK_EQ(state_
, PLAYING
);
382 DCHECK_GT(play_ref_count_
, 0);
383 if (!--play_ref_count_
)
387 void WebRtcAudioRenderer::Stop() {
388 DVLOG(1) << "WebRtcAudioRenderer::Stop()";
389 DCHECK(thread_checker_
.CalledOnValidThread());
391 base::AutoLock
auto_lock(lock_
);
392 if (state_
== UNINITIALIZED
)
395 if (--start_ref_count_
)
398 DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
400 source_
->RemoveAudioRenderer(this);
402 state_
= UNINITIALIZED
;
405 // Make sure to stop the sink while _not_ holding the lock since the Render()
406 // callback may currently be executing and try to grab the lock while we're
407 // stopping the thread on which it runs.
411 void WebRtcAudioRenderer::SetVolume(float volume
) {
412 DCHECK(thread_checker_
.CalledOnValidThread());
413 DCHECK(volume
>= 0.0f
&& volume
<= 1.0f
);
415 playing_state_
.set_volume(volume
);
416 OnPlayStateChanged(media_stream_
, &playing_state_
);
419 void WebRtcAudioRenderer::SwitchOutputDevice(
420 const std::string
& device_id
,
421 const GURL
& security_origin
,
422 const media::SwitchOutputDeviceCB
& callback
) {
423 DCHECK(thread_checker_
.CalledOnValidThread());
425 DVLOG(1) << __FUNCTION__
426 << "(" << device_id
<< ", " << security_origin
<< ")";
427 sink_
->SwitchOutputDevice(device_id
, security_origin
, callback
);
430 base::TimeDelta
WebRtcAudioRenderer::GetCurrentRenderTime() const {
431 DCHECK(thread_checker_
.CalledOnValidThread());
432 base::AutoLock
auto_lock(lock_
);
433 return current_time_
;
436 bool WebRtcAudioRenderer::IsLocalRenderer() const {
440 int WebRtcAudioRenderer::Render(media::AudioBus
* audio_bus
,
441 int audio_delay_milliseconds
) {
442 base::AutoLock
auto_lock(lock_
);
446 DVLOG(2) << "WebRtcAudioRenderer::Render()";
447 DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds
;
449 audio_delay_milliseconds_
= audio_delay_milliseconds
;
452 audio_fifo_
->Consume(audio_bus
, audio_bus
->frames());
454 SourceCallback(0, audio_bus
);
456 return (state_
== PLAYING
) ? audio_bus
->frames() : 0;
459 void WebRtcAudioRenderer::OnRenderError() {
461 LOG(ERROR
) << "OnRenderError()";
464 // Called by AudioPullFifo when more data is necessary.
465 void WebRtcAudioRenderer::SourceCallback(
466 int fifo_frame_delay
, media::AudioBus
* audio_bus
) {
467 base::TimeTicks start_time
= base::TimeTicks::Now() ;
468 DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
469 << fifo_frame_delay
<< ", "
470 << audio_bus
->frames() << ")";
472 int output_delay_milliseconds
= audio_delay_milliseconds_
;
473 output_delay_milliseconds
+= fifo_delay_milliseconds_
;
474 DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds
;
476 // We need to keep render data for the |source_| regardless of |state_|,
477 // otherwise the data will be buffered up inside |source_|.
478 source_
->RenderData(audio_bus
, sink_params_
.sample_rate(),
479 output_delay_milliseconds
,
482 // Avoid filling up the audio bus if we are not playing; instead
483 // return here and ensure that the returned value in Render() is 0.
484 if (state_
!= PLAYING
)
487 if (++render_callback_count_
== kNumCallbacksBetweenRenderTimeHistograms
) {
488 base::TimeDelta elapsed
= base::TimeTicks::Now() - start_time
;
489 render_callback_count_
= 0;
490 UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed
);
494 void WebRtcAudioRenderer::UpdateSourceVolume(
495 webrtc::AudioSourceInterface
* source
) {
496 DCHECK(thread_checker_
.CalledOnValidThread());
498 // Note: If there are no playing audio renderers, then the volume will be
502 SourcePlayingStates::iterator entry
= source_playing_states_
.find(source
);
503 if (entry
!= source_playing_states_
.end()) {
504 PlayingStates
& states
= entry
->second
;
505 for (PlayingStates::const_iterator it
= states
.begin();
506 it
!= states
.end(); ++it
) {
507 if ((*it
)->playing())
508 volume
+= (*it
)->volume();
512 // The valid range for volume scaling of a remote webrtc source is
513 // 0.0-10.0 where 1.0 is no attenuation/boost.
514 DCHECK(volume
>= 0.0f
);
518 DVLOG(1) << "Setting remote source volume: " << volume
;
519 if (!signaling_thread_
->BelongsToCurrentThread()) {
520 // Libjingle hands out proxy objects in most cases, but the audio source
521 // object is an exception (bug?). So, to work around that, we need to make
522 // sure we call SetVolume on the signaling thread.
523 signaling_thread_
->PostTask(FROM_HERE
,
524 base::Bind(&webrtc::AudioSourceInterface::SetVolume
, source
, volume
));
526 source
->SetVolume(volume
);
530 bool WebRtcAudioRenderer::AddPlayingState(
531 webrtc::AudioSourceInterface
* source
,
532 PlayingState
* state
) {
533 DCHECK(thread_checker_
.CalledOnValidThread());
534 DCHECK(state
->playing());
535 // Look up or add the |source| to the map.
536 PlayingStates
& array
= source_playing_states_
[source
];
537 if (std::find(array
.begin(), array
.end(), state
) != array
.end())
540 array
.push_back(state
);
545 bool WebRtcAudioRenderer::RemovePlayingState(
546 webrtc::AudioSourceInterface
* source
,
547 PlayingState
* state
) {
548 DCHECK(thread_checker_
.CalledOnValidThread());
549 DCHECK(!state
->playing());
550 SourcePlayingStates::iterator found
= source_playing_states_
.find(source
);
551 if (found
== source_playing_states_
.end())
554 PlayingStates
& array
= found
->second
;
555 PlayingStates::iterator state_it
=
556 std::find(array
.begin(), array
.end(), state
);
557 if (state_it
== array
.end())
560 array
.erase(state_it
);
563 source_playing_states_
.erase(found
);
568 void WebRtcAudioRenderer::OnPlayStateChanged(
569 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
,
570 PlayingState
* state
) {
571 webrtc::AudioTrackVector
tracks(media_stream
->GetAudioTracks());
572 for (webrtc::AudioTrackVector::iterator it
= tracks
.begin();
573 it
!= tracks
.end(); ++it
) {
574 webrtc::AudioSourceInterface
* source
= (*it
)->GetSource();
576 if (!state
->playing()) {
577 if (RemovePlayingState(source
, state
))
579 } else if (AddPlayingState(source
, state
)) {
582 UpdateSourceVolume(source
);
586 } // namespace content