1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_renderer.h"
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/strings/string_util.h"
10 #include "base/strings/stringprintf.h"
11 #include "content/renderer/media/audio_device_factory.h"
12 #include "content/renderer/media/media_stream_dispatcher.h"
13 #include "content/renderer/media/webrtc_audio_device_impl.h"
14 #include "content/renderer/media/webrtc_logging.h"
15 #include "content/renderer/render_frame_impl.h"
16 #include "media/audio/audio_output_device.h"
17 #include "media/audio/audio_parameters.h"
18 #include "media/audio/sample_rates.h"
19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
24 #include "base/win/windows_version.h"
25 #include "media/audio/win/core_audio_util_win.h"
32 // Supported hardware sample rates for output sides.
33 #if defined(OS_WIN) || defined(OS_MACOSX)
34 // AudioHardwareConfig::GetOutputSampleRate() asks the audio layer for its
35 // current sample rate (set by the user) on Windows and Mac OS X. The listed
36 // rates below adds restrictions and Initialize() will fail if the user selects
37 // any rate outside these ranges.
38 const int kValidOutputRates
[] = {96000, 48000, 44100, 32000, 16000};
39 #elif defined(OS_LINUX) || defined(OS_OPENBSD)
40 const int kValidOutputRates
[] = {48000, 44100};
41 #elif defined(OS_ANDROID)
42 // TODO(leozwang): We want to use native sampling rate on Android to achieve
43 // low latency, currently 16000 is used to work around audio problem on some
45 const int kValidOutputRates
[] = {48000, 44100, 16000};
47 const int kValidOutputRates
[] = {44100};
50 // TODO(xians): Merge the following code to WebRtcAudioCapturer, or remove.
51 enum AudioFramesPerBuffer
{
61 kUnexpectedAudioBufferSize
// Must always be last!
64 // Helper method to convert integral values to their respective enum values
65 // above, or kUnexpectedAudioBufferSize if no match exists.
66 // We map 441 to k440 to avoid changes in the XML part for histograms.
67 // It is still possible to map the histogram result to the actual buffer size.
68 // See http://crbug.com/243450 for details.
69 AudioFramesPerBuffer
AsAudioFramesPerBuffer(int frames_per_buffer
) {
70 switch (frames_per_buffer
) {
71 case 160: return k160
;
72 case 320: return k320
;
73 case 441: return k440
;
74 case 480: return k480
;
75 case 640: return k640
;
76 case 880: return k880
;
77 case 960: return k960
;
78 case 1440: return k1440
;
79 case 1920: return k1920
;
81 return kUnexpectedAudioBufferSize
;
84 void AddHistogramFramesPerBuffer(int param
) {
85 AudioFramesPerBuffer afpb
= AsAudioFramesPerBuffer(param
);
86 if (afpb
!= kUnexpectedAudioBufferSize
) {
87 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputFramesPerBuffer",
88 afpb
, kUnexpectedAudioBufferSize
);
90 // Report unexpected sample rates using a unique histogram name.
91 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputFramesPerBufferUnexpected", param
);
95 // This is a simple wrapper class that's handed out to users of a shared
96 // WebRtcAudioRenderer instance. This class maintains the per-user 'playing'
97 // and 'started' states to avoid problems related to incorrect usage which
98 // might violate the implementation assumptions inside WebRtcAudioRenderer
99 // (see the play reference count).
100 class SharedAudioRenderer
: public MediaStreamAudioRenderer
{
102 // Callback definition for a callback that is called when when Play(), Pause()
103 // or SetVolume are called (whenever the internal |playing_state_| changes).
104 typedef base::Callback
<
105 void(const scoped_refptr
<webrtc::MediaStreamInterface
>&,
106 WebRtcAudioRenderer::PlayingState
*)> OnPlayStateChanged
;
109 const scoped_refptr
<MediaStreamAudioRenderer
>& delegate
,
110 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
,
111 const OnPlayStateChanged
& on_play_state_changed
)
112 : delegate_(delegate
), media_stream_(media_stream
), started_(false),
113 on_play_state_changed_(on_play_state_changed
) {
114 DCHECK(!on_play_state_changed_
.is_null());
115 DCHECK(media_stream_
.get());
119 virtual ~SharedAudioRenderer() {
120 DCHECK(thread_checker_
.CalledOnValidThread());
121 DVLOG(1) << __FUNCTION__
;
125 virtual void Start() OVERRIDE
{
126 DCHECK(thread_checker_
.CalledOnValidThread());
133 virtual void Play() OVERRIDE
{
134 DCHECK(thread_checker_
.CalledOnValidThread());
136 if (playing_state_
.playing())
138 playing_state_
.set_playing(true);
139 on_play_state_changed_
.Run(media_stream_
, &playing_state_
);
142 virtual void Pause() OVERRIDE
{
143 DCHECK(thread_checker_
.CalledOnValidThread());
145 if (!playing_state_
.playing())
147 playing_state_
.set_playing(false);
148 on_play_state_changed_
.Run(media_stream_
, &playing_state_
);
151 virtual void Stop() OVERRIDE
{
152 DCHECK(thread_checker_
.CalledOnValidThread());
160 virtual void SetVolume(float volume
) OVERRIDE
{
161 DCHECK(thread_checker_
.CalledOnValidThread());
162 DCHECK(volume
>= 0.0f
&& volume
<= 1.0f
);
163 playing_state_
.set_volume(volume
);
164 on_play_state_changed_
.Run(media_stream_
, &playing_state_
);
167 virtual base::TimeDelta
GetCurrentRenderTime() const OVERRIDE
{
168 DCHECK(thread_checker_
.CalledOnValidThread());
169 return delegate_
->GetCurrentRenderTime();
172 virtual bool IsLocalRenderer() const OVERRIDE
{
173 DCHECK(thread_checker_
.CalledOnValidThread());
174 return delegate_
->IsLocalRenderer();
178 base::ThreadChecker thread_checker_
;
179 const scoped_refptr
<MediaStreamAudioRenderer
> delegate_
;
180 const scoped_refptr
<webrtc::MediaStreamInterface
> media_stream_
;
182 WebRtcAudioRenderer::PlayingState playing_state_
;
183 OnPlayStateChanged on_play_state_changed_
;
186 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
187 // depending on whether or not an input element is currently open with
189 int GetCurrentDuckingFlag(int render_frame_id
) {
190 RenderFrameImpl
* const frame
=
191 RenderFrameImpl::FromRoutingID(render_frame_id
);
192 MediaStreamDispatcher
* const dispatcher
= frame
?
193 frame
->GetMediaStreamDispatcher() : NULL
;
194 if (dispatcher
&& dispatcher
->IsAudioDuckingActive()) {
195 return media::AudioParameters::DUCKING
;
198 return media::AudioParameters::NO_EFFECTS
;
203 WebRtcAudioRenderer::WebRtcAudioRenderer(
204 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
,
205 int source_render_view_id
,
206 int source_render_frame_id
,
209 int frames_per_buffer
)
210 : state_(UNINITIALIZED
),
211 source_render_view_id_(source_render_view_id
),
212 source_render_frame_id_(source_render_frame_id
),
213 session_id_(session_id
),
214 media_stream_(media_stream
),
218 audio_delay_milliseconds_(0),
219 fifo_delay_milliseconds_(0),
220 sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY
,
221 media::CHANNEL_LAYOUT_STEREO
, 0, sample_rate
, 16,
223 GetCurrentDuckingFlag(source_render_frame_id
)) {
224 WebRtcLogMessage(base::StringPrintf(
225 "WAR::WAR. source_render_view_id=%d"
226 ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
227 source_render_view_id
,
231 sink_params_
.effects()));
234 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
235 DCHECK(thread_checker_
.CalledOnValidThread());
236 DCHECK_EQ(state_
, UNINITIALIZED
);
239 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource
* source
) {
240 DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
241 DCHECK(thread_checker_
.CalledOnValidThread());
242 base::AutoLock
auto_lock(lock_
);
243 DCHECK_EQ(state_
, UNINITIALIZED
);
245 DCHECK(!sink_
.get());
248 // WebRTC does not yet support higher rates than 96000 on the client side
249 // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
250 // we change the rate to 48000 instead. The consequence is that the native
251 // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
252 // which will then be resampled by the audio converted on the browser side
253 // to match the native audio layer.
254 int sample_rate
= sink_params_
.sample_rate();
255 DVLOG(1) << "Audio output hardware sample rate: " << sample_rate
;
256 if (sample_rate
== 192000) {
257 DVLOG(1) << "Resampling from 48000 to 192000 is required";
260 media::AudioSampleRate asr
;
261 if (media::ToAudioSampleRate(sample_rate
, &asr
)) {
262 UMA_HISTOGRAM_ENUMERATION(
263 "WebRTC.AudioOutputSampleRate", asr
, media::kAudioSampleRateMax
+ 1);
265 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
269 // Verify that the reported output hardware sample rate is supported
270 // on the current platform.
271 if (std::find(&kValidOutputRates
[0],
272 &kValidOutputRates
[0] + arraysize(kValidOutputRates
),
274 &kValidOutputRates
[arraysize(kValidOutputRates
)]) {
275 DLOG(ERROR
) << sample_rate
<< " is not a supported output rate.";
279 // Set up audio parameters for the source, i.e., the WebRTC client.
281 // The WebRTC client only supports multiples of 10ms as buffer size where
282 // 10ms is preferred for lowest possible delay.
283 media::AudioParameters source_params
;
284 const int frames_per_10ms
= (sample_rate
/ 100);
285 DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms
;
287 source_params
.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY
,
288 sink_params_
.channel_layout(), sink_params_
.channels(), 0,
289 sample_rate
, 16, frames_per_10ms
);
291 // Update audio parameters for the sink, i.e., the native audio output stream.
292 // We strive to open up using native parameters to achieve best possible
293 // performance and to ensure that no FIFO is needed on the browser side to
294 // match the client request. Any mismatch between the source and the sink is
295 // taken care of in this class instead using a pull FIFO.
297 // Use native output size as default.
298 int frames_per_buffer
= sink_params_
.frames_per_buffer();
299 #if defined(OS_ANDROID)
300 // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
301 // cases. Might not be possible to come up with the perfect solution using
302 // the render side only.
303 if (frames_per_buffer
< 2 * frames_per_10ms
) {
304 // Examples of low-latency frame sizes and the resulting |buffer_size|:
305 // Nexus 7 : 240 audio frames => 2*480 = 960
306 // Nexus 10 : 256 => 2*441 = 882
307 // Galaxy Nexus: 144 => 2*441 = 882
308 frames_per_buffer
= 2 * frames_per_10ms
;
309 DVLOG(1) << "Low-latency output detected on Android";
312 DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer
;
314 sink_params_
.Reset(sink_params_
.format(), sink_params_
.channel_layout(),
315 sink_params_
.channels(), 0, sample_rate
, 16,
318 // Create a FIFO if re-buffering is required to match the source input with
319 // the sink request. The source acts as provider here and the sink as
321 fifo_delay_milliseconds_
= 0;
322 if (source_params
.frames_per_buffer() != sink_params_
.frames_per_buffer()) {
323 DVLOG(1) << "Rebuffering from " << source_params
.frames_per_buffer()
324 << " to " << sink_params_
.frames_per_buffer();
325 audio_fifo_
.reset(new media::AudioPullFifo(
326 source_params
.channels(),
327 source_params
.frames_per_buffer(),
329 &WebRtcAudioRenderer::SourceCallback
,
330 base::Unretained(this))));
332 if (sink_params_
.frames_per_buffer() > source_params
.frames_per_buffer()) {
333 int frame_duration_milliseconds
= base::Time::kMillisecondsPerSecond
/
334 static_cast<double>(source_params
.sample_rate());
335 fifo_delay_milliseconds_
= (sink_params_
.frames_per_buffer() -
336 source_params
.frames_per_buffer()) * frame_duration_milliseconds
;
342 // Configure the audio rendering client and start rendering.
343 sink_
= AudioDeviceFactory::NewOutputDevice(
344 source_render_view_id_
, source_render_frame_id_
);
346 DCHECK_GE(session_id_
, 0);
347 sink_
->InitializeWithSessionId(sink_params_
, this, session_id_
);
351 // User must call Play() before any audio can be heard.
354 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputFramesPerBuffer",
355 source_params
.frames_per_buffer(),
356 kUnexpectedAudioBufferSize
);
357 AddHistogramFramesPerBuffer(source_params
.frames_per_buffer());
362 scoped_refptr
<MediaStreamAudioRenderer
>
363 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
364 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
) {
365 content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed
=
366 base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged
, this);
367 return new SharedAudioRenderer(this, media_stream
, on_play_state_changed
);
370 bool WebRtcAudioRenderer::IsStarted() const {
371 DCHECK(thread_checker_
.CalledOnValidThread());
372 return start_ref_count_
!= 0;
375 void WebRtcAudioRenderer::Start() {
376 DVLOG(1) << "WebRtcAudioRenderer::Start()";
377 DCHECK(thread_checker_
.CalledOnValidThread());
381 void WebRtcAudioRenderer::Play() {
382 DVLOG(1) << "WebRtcAudioRenderer::Play()";
383 DCHECK(thread_checker_
.CalledOnValidThread());
385 if (playing_state_
.playing())
388 playing_state_
.set_playing(true);
390 OnPlayStateChanged(media_stream_
, &playing_state_
);
393 void WebRtcAudioRenderer::EnterPlayState() {
394 DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
395 DCHECK(thread_checker_
.CalledOnValidThread());
396 DCHECK_GT(start_ref_count_
, 0) << "Did you forget to call Start()?";
397 base::AutoLock
auto_lock(lock_
);
398 if (state_
== UNINITIALIZED
)
401 DCHECK(play_ref_count_
== 0 || state_
== PLAYING
);
404 if (state_
!= PLAYING
) {
408 audio_delay_milliseconds_
= 0;
409 audio_fifo_
->Clear();
414 void WebRtcAudioRenderer::Pause() {
415 DVLOG(1) << "WebRtcAudioRenderer::Pause()";
416 DCHECK(thread_checker_
.CalledOnValidThread());
417 if (!playing_state_
.playing())
420 playing_state_
.set_playing(false);
422 OnPlayStateChanged(media_stream_
, &playing_state_
);
425 void WebRtcAudioRenderer::EnterPauseState() {
426 DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
427 DCHECK(thread_checker_
.CalledOnValidThread());
428 DCHECK_GT(start_ref_count_
, 0) << "Did you forget to call Start()?";
429 base::AutoLock
auto_lock(lock_
);
430 if (state_
== UNINITIALIZED
)
433 DCHECK_EQ(state_
, PLAYING
);
434 DCHECK_GT(play_ref_count_
, 0);
435 if (!--play_ref_count_
)
439 void WebRtcAudioRenderer::Stop() {
440 DVLOG(1) << "WebRtcAudioRenderer::Stop()";
441 DCHECK(thread_checker_
.CalledOnValidThread());
443 base::AutoLock
auto_lock(lock_
);
444 if (state_
== UNINITIALIZED
)
447 if (--start_ref_count_
)
450 DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
452 source_
->RemoveAudioRenderer(this);
454 state_
= UNINITIALIZED
;
457 // Make sure to stop the sink while _not_ holding the lock since the Render()
458 // callback may currently be executing and try to grab the lock while we're
459 // stopping the thread on which it runs.
463 void WebRtcAudioRenderer::SetVolume(float volume
) {
464 DCHECK(thread_checker_
.CalledOnValidThread());
465 DCHECK(volume
>= 0.0f
&& volume
<= 1.0f
);
467 playing_state_
.set_volume(volume
);
468 OnPlayStateChanged(media_stream_
, &playing_state_
);
471 base::TimeDelta
WebRtcAudioRenderer::GetCurrentRenderTime() const {
472 DCHECK(thread_checker_
.CalledOnValidThread());
473 base::AutoLock
auto_lock(lock_
);
474 return current_time_
;
477 bool WebRtcAudioRenderer::IsLocalRenderer() const {
481 int WebRtcAudioRenderer::Render(media::AudioBus
* audio_bus
,
482 int audio_delay_milliseconds
) {
483 base::AutoLock
auto_lock(lock_
);
487 DVLOG(2) << "WebRtcAudioRenderer::Render()";
488 DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds
;
490 audio_delay_milliseconds_
= audio_delay_milliseconds
;
493 audio_fifo_
->Consume(audio_bus
, audio_bus
->frames());
495 SourceCallback(0, audio_bus
);
497 return (state_
== PLAYING
) ? audio_bus
->frames() : 0;
500 void WebRtcAudioRenderer::OnRenderError() {
502 LOG(ERROR
) << "OnRenderError()";
505 // Called by AudioPullFifo when more data is necessary.
506 void WebRtcAudioRenderer::SourceCallback(
507 int fifo_frame_delay
, media::AudioBus
* audio_bus
) {
508 DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
509 << fifo_frame_delay
<< ", "
510 << audio_bus
->frames() << ")";
512 int output_delay_milliseconds
= audio_delay_milliseconds_
;
513 output_delay_milliseconds
+= fifo_delay_milliseconds_
;
514 DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds
;
516 // We need to keep render data for the |source_| regardless of |state_|,
517 // otherwise the data will be buffered up inside |source_|.
518 source_
->RenderData(audio_bus
, sink_params_
.sample_rate(),
519 output_delay_milliseconds
,
522 // Avoid filling up the audio bus if we are not playing; instead
523 // return here and ensure that the returned value in Render() is 0.
524 if (state_
!= PLAYING
)
528 void WebRtcAudioRenderer::UpdateSourceVolume(
529 webrtc::AudioSourceInterface
* source
) {
530 DCHECK(thread_checker_
.CalledOnValidThread());
532 // Note: If there are no playing audio renderers, then the volume will be
536 SourcePlayingStates::iterator entry
= source_playing_states_
.find(source
);
537 if (entry
!= source_playing_states_
.end()) {
538 PlayingStates
& states
= entry
->second
;
539 for (PlayingStates::const_iterator it
= states
.begin();
540 it
!= states
.end(); ++it
) {
541 if ((*it
)->playing())
542 volume
+= (*it
)->volume();
546 // The valid range for volume scaling of a remote webrtc source is
547 // 0.0-10.0 where 1.0 is no attenuation/boost.
548 DCHECK(volume
>= 0.0f
);
552 DVLOG(1) << "Setting remote source volume: " << volume
;
553 source
->SetVolume(volume
);
556 bool WebRtcAudioRenderer::AddPlayingState(
557 webrtc::AudioSourceInterface
* source
,
558 PlayingState
* state
) {
559 DCHECK(thread_checker_
.CalledOnValidThread());
560 DCHECK(state
->playing());
561 // Look up or add the |source| to the map.
562 PlayingStates
& array
= source_playing_states_
[source
];
563 if (std::find(array
.begin(), array
.end(), state
) != array
.end())
566 array
.push_back(state
);
571 bool WebRtcAudioRenderer::RemovePlayingState(
572 webrtc::AudioSourceInterface
* source
,
573 PlayingState
* state
) {
574 DCHECK(thread_checker_
.CalledOnValidThread());
575 DCHECK(!state
->playing());
576 SourcePlayingStates::iterator found
= source_playing_states_
.find(source
);
577 if (found
== source_playing_states_
.end())
580 PlayingStates
& array
= found
->second
;
581 PlayingStates::iterator state_it
=
582 std::find(array
.begin(), array
.end(), state
);
583 if (state_it
== array
.end())
586 array
.erase(state_it
);
589 source_playing_states_
.erase(found
);
594 void WebRtcAudioRenderer::OnPlayStateChanged(
595 const scoped_refptr
<webrtc::MediaStreamInterface
>& media_stream
,
596 PlayingState
* state
) {
597 webrtc::AudioTrackVector
tracks(media_stream
->GetAudioTracks());
598 for (webrtc::AudioTrackVector::iterator it
= tracks
.begin();
599 it
!= tracks
.end(); ++it
) {
600 webrtc::AudioSourceInterface
* source
= (*it
)->GetSource();
602 if (!state
->playing()) {
603 if (RemovePlayingState(source
, state
))
605 } else if (AddPlayingState(source
, state
)) {
608 UpdateSourceVolume(source
);
612 } // namespace content