Updating trunk VERSION from 2139.0 to 2140.0
[chromium-blink-merge.git] / content / renderer / media / webrtc_audio_renderer.cc
blob006f12fe4f7b12d8fdc085ef813b780987933bc9
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_renderer.h"
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/strings/string_util.h"
10 #include "base/strings/stringprintf.h"
11 #include "content/renderer/media/audio_device_factory.h"
12 #include "content/renderer/media/media_stream_dispatcher.h"
13 #include "content/renderer/media/webrtc_audio_device_impl.h"
14 #include "content/renderer/media/webrtc_logging.h"
15 #include "content/renderer/render_frame_impl.h"
16 #include "media/audio/audio_output_device.h"
17 #include "media/audio/audio_parameters.h"
18 #include "media/audio/sample_rates.h"
19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
23 #if defined(OS_WIN)
24 #include "base/win/windows_version.h"
25 #include "media/audio/win/core_audio_util_win.h"
26 #endif
28 namespace content {
30 namespace {
32 // Supported hardware sample rates for output sides.
33 #if defined(OS_WIN) || defined(OS_MACOSX)
34 // AudioHardwareConfig::GetOutputSampleRate() asks the audio layer for its
35 // current sample rate (set by the user) on Windows and Mac OS X. The listed
36 // rates below adds restrictions and Initialize() will fail if the user selects
37 // any rate outside these ranges.
38 const int kValidOutputRates[] = {96000, 48000, 44100, 32000, 16000};
39 #elif defined(OS_LINUX) || defined(OS_OPENBSD)
40 const int kValidOutputRates[] = {48000, 44100};
41 #elif defined(OS_ANDROID)
42 // TODO(leozwang): We want to use native sampling rate on Android to achieve
43 // low latency, currently 16000 is used to work around audio problem on some
44 // Android devices.
45 const int kValidOutputRates[] = {48000, 44100, 16000};
46 #else
47 const int kValidOutputRates[] = {44100};
48 #endif
50 // TODO(xians): Merge the following code to WebRtcAudioCapturer, or remove.
51 enum AudioFramesPerBuffer {
52 k160,
53 k320,
54 k440,
55 k480,
56 k640,
57 k880,
58 k960,
59 k1440,
60 k1920,
61 kUnexpectedAudioBufferSize // Must always be last!
64 // Helper method to convert integral values to their respective enum values
65 // above, or kUnexpectedAudioBufferSize if no match exists.
66 // We map 441 to k440 to avoid changes in the XML part for histograms.
67 // It is still possible to map the histogram result to the actual buffer size.
68 // See http://crbug.com/243450 for details.
69 AudioFramesPerBuffer AsAudioFramesPerBuffer(int frames_per_buffer) {
70 switch (frames_per_buffer) {
71 case 160: return k160;
72 case 320: return k320;
73 case 441: return k440;
74 case 480: return k480;
75 case 640: return k640;
76 case 880: return k880;
77 case 960: return k960;
78 case 1440: return k1440;
79 case 1920: return k1920;
81 return kUnexpectedAudioBufferSize;
84 void AddHistogramFramesPerBuffer(int param) {
85 AudioFramesPerBuffer afpb = AsAudioFramesPerBuffer(param);
86 if (afpb != kUnexpectedAudioBufferSize) {
87 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputFramesPerBuffer",
88 afpb, kUnexpectedAudioBufferSize);
89 } else {
90 // Report unexpected sample rates using a unique histogram name.
91 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputFramesPerBufferUnexpected", param);
95 // This is a simple wrapper class that's handed out to users of a shared
96 // WebRtcAudioRenderer instance. This class maintains the per-user 'playing'
97 // and 'started' states to avoid problems related to incorrect usage which
98 // might violate the implementation assumptions inside WebRtcAudioRenderer
99 // (see the play reference count).
100 class SharedAudioRenderer : public MediaStreamAudioRenderer {
101 public:
102 // Callback definition for a callback that is called when when Play(), Pause()
103 // or SetVolume are called (whenever the internal |playing_state_| changes).
104 typedef base::Callback<
105 void(const scoped_refptr<webrtc::MediaStreamInterface>&,
106 WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
108 SharedAudioRenderer(
109 const scoped_refptr<MediaStreamAudioRenderer>& delegate,
110 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
111 const OnPlayStateChanged& on_play_state_changed)
112 : delegate_(delegate), media_stream_(media_stream), started_(false),
113 on_play_state_changed_(on_play_state_changed) {
114 DCHECK(!on_play_state_changed_.is_null());
115 DCHECK(media_stream_.get());
118 protected:
119 virtual ~SharedAudioRenderer() {
120 DCHECK(thread_checker_.CalledOnValidThread());
121 DVLOG(1) << __FUNCTION__;
122 Stop();
125 virtual void Start() OVERRIDE {
126 DCHECK(thread_checker_.CalledOnValidThread());
127 if (started_)
128 return;
129 started_ = true;
130 delegate_->Start();
133 virtual void Play() OVERRIDE {
134 DCHECK(thread_checker_.CalledOnValidThread());
135 DCHECK(started_);
136 if (playing_state_.playing())
137 return;
138 playing_state_.set_playing(true);
139 on_play_state_changed_.Run(media_stream_, &playing_state_);
142 virtual void Pause() OVERRIDE {
143 DCHECK(thread_checker_.CalledOnValidThread());
144 DCHECK(started_);
145 if (!playing_state_.playing())
146 return;
147 playing_state_.set_playing(false);
148 on_play_state_changed_.Run(media_stream_, &playing_state_);
151 virtual void Stop() OVERRIDE {
152 DCHECK(thread_checker_.CalledOnValidThread());
153 if (!started_)
154 return;
155 Pause();
156 started_ = false;
157 delegate_->Stop();
160 virtual void SetVolume(float volume) OVERRIDE {
161 DCHECK(thread_checker_.CalledOnValidThread());
162 DCHECK(volume >= 0.0f && volume <= 1.0f);
163 playing_state_.set_volume(volume);
164 on_play_state_changed_.Run(media_stream_, &playing_state_);
167 virtual base::TimeDelta GetCurrentRenderTime() const OVERRIDE {
168 DCHECK(thread_checker_.CalledOnValidThread());
169 return delegate_->GetCurrentRenderTime();
172 virtual bool IsLocalRenderer() const OVERRIDE {
173 DCHECK(thread_checker_.CalledOnValidThread());
174 return delegate_->IsLocalRenderer();
177 private:
178 base::ThreadChecker thread_checker_;
179 const scoped_refptr<MediaStreamAudioRenderer> delegate_;
180 const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
181 bool started_;
182 WebRtcAudioRenderer::PlayingState playing_state_;
183 OnPlayStateChanged on_play_state_changed_;
186 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
187 // depending on whether or not an input element is currently open with
188 // ducking enabled.
189 int GetCurrentDuckingFlag(int render_frame_id) {
190 RenderFrameImpl* const frame =
191 RenderFrameImpl::FromRoutingID(render_frame_id);
192 MediaStreamDispatcher* const dispatcher = frame ?
193 frame->GetMediaStreamDispatcher() : NULL;
194 if (dispatcher && dispatcher->IsAudioDuckingActive()) {
195 return media::AudioParameters::DUCKING;
198 return media::AudioParameters::NO_EFFECTS;
201 } // namespace
203 WebRtcAudioRenderer::WebRtcAudioRenderer(
204 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
205 int source_render_view_id,
206 int source_render_frame_id,
207 int session_id,
208 int sample_rate,
209 int frames_per_buffer)
210 : state_(UNINITIALIZED),
211 source_render_view_id_(source_render_view_id),
212 source_render_frame_id_(source_render_frame_id),
213 session_id_(session_id),
214 media_stream_(media_stream),
215 source_(NULL),
216 play_ref_count_(0),
217 start_ref_count_(0),
218 audio_delay_milliseconds_(0),
219 fifo_delay_milliseconds_(0),
220 sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
221 media::CHANNEL_LAYOUT_STEREO, 0, sample_rate, 16,
222 frames_per_buffer,
223 GetCurrentDuckingFlag(source_render_frame_id)) {
224 WebRtcLogMessage(base::StringPrintf(
225 "WAR::WAR. source_render_view_id=%d"
226 ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
227 source_render_view_id,
228 session_id,
229 sample_rate,
230 frames_per_buffer,
231 sink_params_.effects()));
234 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
235 DCHECK(thread_checker_.CalledOnValidThread());
236 DCHECK_EQ(state_, UNINITIALIZED);
239 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
240 DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
241 DCHECK(thread_checker_.CalledOnValidThread());
242 base::AutoLock auto_lock(lock_);
243 DCHECK_EQ(state_, UNINITIALIZED);
244 DCHECK(source);
245 DCHECK(!sink_.get());
246 DCHECK(!source_);
248 // WebRTC does not yet support higher rates than 96000 on the client side
249 // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
250 // we change the rate to 48000 instead. The consequence is that the native
251 // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
252 // which will then be resampled by the audio converted on the browser side
253 // to match the native audio layer.
254 int sample_rate = sink_params_.sample_rate();
255 DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
256 if (sample_rate == 192000) {
257 DVLOG(1) << "Resampling from 48000 to 192000 is required";
258 sample_rate = 48000;
260 media::AudioSampleRate asr;
261 if (media::ToAudioSampleRate(sample_rate, &asr)) {
262 UMA_HISTOGRAM_ENUMERATION(
263 "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
264 } else {
265 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
266 sample_rate);
269 // Verify that the reported output hardware sample rate is supported
270 // on the current platform.
271 if (std::find(&kValidOutputRates[0],
272 &kValidOutputRates[0] + arraysize(kValidOutputRates),
273 sample_rate) ==
274 &kValidOutputRates[arraysize(kValidOutputRates)]) {
275 DLOG(ERROR) << sample_rate << " is not a supported output rate.";
276 return false;
279 // Set up audio parameters for the source, i.e., the WebRTC client.
281 // The WebRTC client only supports multiples of 10ms as buffer size where
282 // 10ms is preferred for lowest possible delay.
283 media::AudioParameters source_params;
284 const int frames_per_10ms = (sample_rate / 100);
285 DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
287 source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
288 sink_params_.channel_layout(), sink_params_.channels(), 0,
289 sample_rate, 16, frames_per_10ms);
291 // Update audio parameters for the sink, i.e., the native audio output stream.
292 // We strive to open up using native parameters to achieve best possible
293 // performance and to ensure that no FIFO is needed on the browser side to
294 // match the client request. Any mismatch between the source and the sink is
295 // taken care of in this class instead using a pull FIFO.
297 // Use native output size as default.
298 int frames_per_buffer = sink_params_.frames_per_buffer();
299 #if defined(OS_ANDROID)
300 // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
301 // cases. Might not be possible to come up with the perfect solution using
302 // the render side only.
303 if (frames_per_buffer < 2 * frames_per_10ms) {
304 // Examples of low-latency frame sizes and the resulting |buffer_size|:
305 // Nexus 7 : 240 audio frames => 2*480 = 960
306 // Nexus 10 : 256 => 2*441 = 882
307 // Galaxy Nexus: 144 => 2*441 = 882
308 frames_per_buffer = 2 * frames_per_10ms;
309 DVLOG(1) << "Low-latency output detected on Android";
311 #endif
312 DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
314 sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
315 sink_params_.channels(), 0, sample_rate, 16,
316 frames_per_buffer);
318 // Create a FIFO if re-buffering is required to match the source input with
319 // the sink request. The source acts as provider here and the sink as
320 // consumer.
321 fifo_delay_milliseconds_ = 0;
322 if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
323 DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
324 << " to " << sink_params_.frames_per_buffer();
325 audio_fifo_.reset(new media::AudioPullFifo(
326 source_params.channels(),
327 source_params.frames_per_buffer(),
328 base::Bind(
329 &WebRtcAudioRenderer::SourceCallback,
330 base::Unretained(this))));
332 if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
333 int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
334 static_cast<double>(source_params.sample_rate());
335 fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
336 source_params.frames_per_buffer()) * frame_duration_milliseconds;
340 source_ = source;
342 // Configure the audio rendering client and start rendering.
343 sink_ = AudioDeviceFactory::NewOutputDevice(
344 source_render_view_id_, source_render_frame_id_);
346 DCHECK_GE(session_id_, 0);
347 sink_->InitializeWithSessionId(sink_params_, this, session_id_);
349 sink_->Start();
351 // User must call Play() before any audio can be heard.
352 state_ = PAUSED;
354 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioOutputFramesPerBuffer",
355 source_params.frames_per_buffer(),
356 kUnexpectedAudioBufferSize);
357 AddHistogramFramesPerBuffer(source_params.frames_per_buffer());
359 return true;
362 scoped_refptr<MediaStreamAudioRenderer>
363 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
364 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
365 content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
366 base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
367 return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
370 bool WebRtcAudioRenderer::IsStarted() const {
371 DCHECK(thread_checker_.CalledOnValidThread());
372 return start_ref_count_ != 0;
375 void WebRtcAudioRenderer::Start() {
376 DVLOG(1) << "WebRtcAudioRenderer::Start()";
377 DCHECK(thread_checker_.CalledOnValidThread());
378 ++start_ref_count_;
381 void WebRtcAudioRenderer::Play() {
382 DVLOG(1) << "WebRtcAudioRenderer::Play()";
383 DCHECK(thread_checker_.CalledOnValidThread());
385 if (playing_state_.playing())
386 return;
388 playing_state_.set_playing(true);
390 OnPlayStateChanged(media_stream_, &playing_state_);
393 void WebRtcAudioRenderer::EnterPlayState() {
394 DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
395 DCHECK(thread_checker_.CalledOnValidThread());
396 DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
397 base::AutoLock auto_lock(lock_);
398 if (state_ == UNINITIALIZED)
399 return;
401 DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
402 ++play_ref_count_;
404 if (state_ != PLAYING) {
405 state_ = PLAYING;
407 if (audio_fifo_) {
408 audio_delay_milliseconds_ = 0;
409 audio_fifo_->Clear();
414 void WebRtcAudioRenderer::Pause() {
415 DVLOG(1) << "WebRtcAudioRenderer::Pause()";
416 DCHECK(thread_checker_.CalledOnValidThread());
417 if (!playing_state_.playing())
418 return;
420 playing_state_.set_playing(false);
422 OnPlayStateChanged(media_stream_, &playing_state_);
425 void WebRtcAudioRenderer::EnterPauseState() {
426 DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
427 DCHECK(thread_checker_.CalledOnValidThread());
428 DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
429 base::AutoLock auto_lock(lock_);
430 if (state_ == UNINITIALIZED)
431 return;
433 DCHECK_EQ(state_, PLAYING);
434 DCHECK_GT(play_ref_count_, 0);
435 if (!--play_ref_count_)
436 state_ = PAUSED;
439 void WebRtcAudioRenderer::Stop() {
440 DVLOG(1) << "WebRtcAudioRenderer::Stop()";
441 DCHECK(thread_checker_.CalledOnValidThread());
443 base::AutoLock auto_lock(lock_);
444 if (state_ == UNINITIALIZED)
445 return;
447 if (--start_ref_count_)
448 return;
450 DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
452 source_->RemoveAudioRenderer(this);
453 source_ = NULL;
454 state_ = UNINITIALIZED;
457 // Make sure to stop the sink while _not_ holding the lock since the Render()
458 // callback may currently be executing and try to grab the lock while we're
459 // stopping the thread on which it runs.
460 sink_->Stop();
463 void WebRtcAudioRenderer::SetVolume(float volume) {
464 DCHECK(thread_checker_.CalledOnValidThread());
465 DCHECK(volume >= 0.0f && volume <= 1.0f);
467 playing_state_.set_volume(volume);
468 OnPlayStateChanged(media_stream_, &playing_state_);
471 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
472 DCHECK(thread_checker_.CalledOnValidThread());
473 base::AutoLock auto_lock(lock_);
474 return current_time_;
477 bool WebRtcAudioRenderer::IsLocalRenderer() const {
478 return false;
481 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
482 int audio_delay_milliseconds) {
483 base::AutoLock auto_lock(lock_);
484 if (!source_)
485 return 0;
487 DVLOG(2) << "WebRtcAudioRenderer::Render()";
488 DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
490 audio_delay_milliseconds_ = audio_delay_milliseconds;
492 if (audio_fifo_)
493 audio_fifo_->Consume(audio_bus, audio_bus->frames());
494 else
495 SourceCallback(0, audio_bus);
497 return (state_ == PLAYING) ? audio_bus->frames() : 0;
500 void WebRtcAudioRenderer::OnRenderError() {
501 NOTIMPLEMENTED();
502 LOG(ERROR) << "OnRenderError()";
505 // Called by AudioPullFifo when more data is necessary.
506 void WebRtcAudioRenderer::SourceCallback(
507 int fifo_frame_delay, media::AudioBus* audio_bus) {
508 DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
509 << fifo_frame_delay << ", "
510 << audio_bus->frames() << ")";
512 int output_delay_milliseconds = audio_delay_milliseconds_;
513 output_delay_milliseconds += fifo_delay_milliseconds_;
514 DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
516 // We need to keep render data for the |source_| regardless of |state_|,
517 // otherwise the data will be buffered up inside |source_|.
518 source_->RenderData(audio_bus, sink_params_.sample_rate(),
519 output_delay_milliseconds,
520 &current_time_);
522 // Avoid filling up the audio bus if we are not playing; instead
523 // return here and ensure that the returned value in Render() is 0.
524 if (state_ != PLAYING)
525 audio_bus->Zero();
528 void WebRtcAudioRenderer::UpdateSourceVolume(
529 webrtc::AudioSourceInterface* source) {
530 DCHECK(thread_checker_.CalledOnValidThread());
532 // Note: If there are no playing audio renderers, then the volume will be
533 // set to 0.0.
534 float volume = 0.0f;
536 SourcePlayingStates::iterator entry = source_playing_states_.find(source);
537 if (entry != source_playing_states_.end()) {
538 PlayingStates& states = entry->second;
539 for (PlayingStates::const_iterator it = states.begin();
540 it != states.end(); ++it) {
541 if ((*it)->playing())
542 volume += (*it)->volume();
546 // The valid range for volume scaling of a remote webrtc source is
547 // 0.0-10.0 where 1.0 is no attenuation/boost.
548 DCHECK(volume >= 0.0f);
549 if (volume > 10.0f)
550 volume = 10.0f;
552 DVLOG(1) << "Setting remote source volume: " << volume;
553 source->SetVolume(volume);
556 bool WebRtcAudioRenderer::AddPlayingState(
557 webrtc::AudioSourceInterface* source,
558 PlayingState* state) {
559 DCHECK(thread_checker_.CalledOnValidThread());
560 DCHECK(state->playing());
561 // Look up or add the |source| to the map.
562 PlayingStates& array = source_playing_states_[source];
563 if (std::find(array.begin(), array.end(), state) != array.end())
564 return false;
566 array.push_back(state);
568 return true;
571 bool WebRtcAudioRenderer::RemovePlayingState(
572 webrtc::AudioSourceInterface* source,
573 PlayingState* state) {
574 DCHECK(thread_checker_.CalledOnValidThread());
575 DCHECK(!state->playing());
576 SourcePlayingStates::iterator found = source_playing_states_.find(source);
577 if (found == source_playing_states_.end())
578 return false;
580 PlayingStates& array = found->second;
581 PlayingStates::iterator state_it =
582 std::find(array.begin(), array.end(), state);
583 if (state_it == array.end())
584 return false;
586 array.erase(state_it);
588 if (array.empty())
589 source_playing_states_.erase(found);
591 return true;
594 void WebRtcAudioRenderer::OnPlayStateChanged(
595 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
596 PlayingState* state) {
597 webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
598 for (webrtc::AudioTrackVector::iterator it = tracks.begin();
599 it != tracks.end(); ++it) {
600 webrtc::AudioSourceInterface* source = (*it)->GetSource();
601 DCHECK(source);
602 if (!state->playing()) {
603 if (RemovePlayingState(source, state))
604 EnterPauseState();
605 } else if (AddPlayingState(source, state)) {
606 EnterPlayState();
608 UpdateSourceVolume(source);
612 } // namespace content