Roll src/third_party/WebKit d10c917:a1123a1 (svn 198729:198730)
[chromium-blink-merge.git] / content / renderer / media / webrtc_audio_renderer.cc
blob23090d1b076890b0a62a66fcf2ab277f427df2c3
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_renderer.h"
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/strings/string_util.h"
10 #include "base/strings/stringprintf.h"
11 #include "content/renderer/media/audio_device_factory.h"
12 #include "content/renderer/media/media_stream_dispatcher.h"
13 #include "content/renderer/media/webrtc_audio_device_impl.h"
14 #include "content/renderer/media/webrtc_logging.h"
15 #include "content/renderer/render_frame_impl.h"
16 #include "media/audio/audio_output_device.h"
17 #include "media/audio/audio_parameters.h"
18 #include "media/audio/sample_rates.h"
19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
23 #if defined(OS_WIN)
24 #include "base/win/windows_version.h"
25 #include "media/audio/win/core_audio_util_win.h"
26 #endif
28 namespace content {
30 namespace {
32 // We add a UMA histogram measuring the execution time of the Render() method
33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
34 // between each callback leads to one UMA update each 100ms.
35 const int kNumCallbacksBetweenRenderTimeHistograms = 10;
37 // This is a simple wrapper class that's handed out to users of a shared
38 // WebRtcAudioRenderer instance. This class maintains the per-user 'playing'
39 // and 'started' states to avoid problems related to incorrect usage which
40 // might violate the implementation assumptions inside WebRtcAudioRenderer
41 // (see the play reference count).
42 class SharedAudioRenderer : public MediaStreamAudioRenderer {
43 public:
44 // Callback definition for a callback that is called when when Play(), Pause()
45 // or SetVolume are called (whenever the internal |playing_state_| changes).
46 typedef base::Callback<
47 void(const scoped_refptr<webrtc::MediaStreamInterface>&,
48 WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
50 SharedAudioRenderer(
51 const scoped_refptr<MediaStreamAudioRenderer>& delegate,
52 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
53 const OnPlayStateChanged& on_play_state_changed)
54 : delegate_(delegate), media_stream_(media_stream), started_(false),
55 on_play_state_changed_(on_play_state_changed) {
56 DCHECK(!on_play_state_changed_.is_null());
57 DCHECK(media_stream_.get());
60 protected:
61 ~SharedAudioRenderer() override {
62 DCHECK(thread_checker_.CalledOnValidThread());
63 DVLOG(1) << __FUNCTION__;
64 Stop();
67 void Start() override {
68 DCHECK(thread_checker_.CalledOnValidThread());
69 if (started_)
70 return;
71 started_ = true;
72 delegate_->Start();
75 void Play() override {
76 DCHECK(thread_checker_.CalledOnValidThread());
77 DCHECK(started_);
78 if (playing_state_.playing())
79 return;
80 playing_state_.set_playing(true);
81 on_play_state_changed_.Run(media_stream_, &playing_state_);
84 void Pause() override {
85 DCHECK(thread_checker_.CalledOnValidThread());
86 DCHECK(started_);
87 if (!playing_state_.playing())
88 return;
89 playing_state_.set_playing(false);
90 on_play_state_changed_.Run(media_stream_, &playing_state_);
93 void Stop() override {
94 DCHECK(thread_checker_.CalledOnValidThread());
95 if (!started_)
96 return;
97 Pause();
98 started_ = false;
99 delegate_->Stop();
102 void SetVolume(float volume) override {
103 DCHECK(thread_checker_.CalledOnValidThread());
104 DCHECK(volume >= 0.0f && volume <= 1.0f);
105 playing_state_.set_volume(volume);
106 on_play_state_changed_.Run(media_stream_, &playing_state_);
109 void SwitchOutputDevice(
110 const std::string& device_id,
111 const GURL& security_origin,
112 const media::SwitchOutputDeviceCB& callback) override {
113 DCHECK(thread_checker_.CalledOnValidThread());
114 DVLOG(1) << __FUNCTION__
115 << "(" << device_id << ", " << security_origin << ")";
116 delegate_->SwitchOutputDevice(device_id, security_origin, callback);
119 base::TimeDelta GetCurrentRenderTime() const override {
120 DCHECK(thread_checker_.CalledOnValidThread());
121 return delegate_->GetCurrentRenderTime();
124 bool IsLocalRenderer() const override {
125 DCHECK(thread_checker_.CalledOnValidThread());
126 return delegate_->IsLocalRenderer();
129 private:
130 base::ThreadChecker thread_checker_;
131 const scoped_refptr<MediaStreamAudioRenderer> delegate_;
132 const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
133 bool started_;
134 WebRtcAudioRenderer::PlayingState playing_state_;
135 OnPlayStateChanged on_play_state_changed_;
138 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
139 // depending on whether or not an input element is currently open with
140 // ducking enabled.
141 int GetCurrentDuckingFlag(int render_frame_id) {
142 RenderFrameImpl* const frame =
143 RenderFrameImpl::FromRoutingID(render_frame_id);
144 MediaStreamDispatcher* const dispatcher = frame ?
145 frame->GetMediaStreamDispatcher() : NULL;
146 if (dispatcher && dispatcher->IsAudioDuckingActive()) {
147 return media::AudioParameters::DUCKING;
150 return media::AudioParameters::NO_EFFECTS;
153 } // namespace
155 int WebRtcAudioRenderer::GetOptimalBufferSize(int sample_rate,
156 int hardware_buffer_size) {
157 // Use native hardware buffer size as default. On Windows, we strive to open
158 // up using this native hardware buffer size to achieve best
159 // possible performance and to ensure that no FIFO is needed on the browser
160 // side to match the client request. That is why there is no #if case for
161 // Windows below.
162 int frames_per_buffer = hardware_buffer_size;
164 #if defined(OS_LINUX) || defined(OS_MACOSX)
165 // On Linux and MacOS, the low level IO implementations on the browser side
166 // supports all buffer size the clients want. We use the native peer
167 // connection buffer size (10ms) to achieve best possible performance.
168 frames_per_buffer = sample_rate / 100;
169 #elif defined(OS_ANDROID)
170 // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
171 // cases. Might not be possible to come up with the perfect solution using
172 // the render side only.
173 int frames_per_10ms = sample_rate / 100;
174 if (frames_per_buffer < 2 * frames_per_10ms) {
175 // Examples of low-latency frame sizes and the resulting |buffer_size|:
176 // Nexus 7 : 240 audio frames => 2*480 = 960
177 // Nexus 10 : 256 => 2*441 = 882
178 // Galaxy Nexus: 144 => 2*441 = 882
179 frames_per_buffer = 2 * frames_per_10ms;
180 DVLOG(1) << "Low-latency output detected on Android";
182 #endif
184 DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
185 return frames_per_buffer;
188 WebRtcAudioRenderer::WebRtcAudioRenderer(
189 const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread,
190 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
191 int source_render_frame_id,
192 int session_id,
193 int sample_rate,
194 int frames_per_buffer)
195 : state_(UNINITIALIZED),
196 source_render_frame_id_(source_render_frame_id),
197 session_id_(session_id),
198 signaling_thread_(signaling_thread),
199 media_stream_(media_stream),
200 source_(NULL),
201 play_ref_count_(0),
202 start_ref_count_(0),
203 audio_delay_milliseconds_(0),
204 fifo_delay_milliseconds_(0),
205 sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
206 media::CHANNEL_LAYOUT_STEREO, sample_rate, 16,
207 frames_per_buffer,
208 GetCurrentDuckingFlag(source_render_frame_id)),
209 render_callback_count_(0) {
210 WebRtcLogMessage(base::StringPrintf(
211 "WAR::WAR. source_render_frame_id=%d"
212 ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
213 source_render_frame_id, session_id, sample_rate, frames_per_buffer,
214 sink_params_.effects()));
217 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
218 DCHECK(thread_checker_.CalledOnValidThread());
219 DCHECK_EQ(state_, UNINITIALIZED);
222 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
223 DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
224 DCHECK(thread_checker_.CalledOnValidThread());
225 base::AutoLock auto_lock(lock_);
226 DCHECK_EQ(state_, UNINITIALIZED);
227 DCHECK(source);
228 DCHECK(!sink_.get());
229 DCHECK(!source_);
231 // WebRTC does not yet support higher rates than 96000 on the client side
232 // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
233 // we change the rate to 48000 instead. The consequence is that the native
234 // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
235 // which will then be resampled by the audio converted on the browser side
236 // to match the native audio layer.
237 int sample_rate = sink_params_.sample_rate();
238 DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
239 if (sample_rate == 192000) {
240 DVLOG(1) << "Resampling from 48000 to 192000 is required";
241 sample_rate = 48000;
243 media::AudioSampleRate asr;
244 if (media::ToAudioSampleRate(sample_rate, &asr)) {
245 UMA_HISTOGRAM_ENUMERATION(
246 "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
247 } else {
248 UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
249 sample_rate);
252 // Set up audio parameters for the source, i.e., the WebRTC client.
254 // The WebRTC client only supports multiples of 10ms as buffer size where
255 // 10ms is preferred for lowest possible delay.
256 media::AudioParameters source_params;
257 const int frames_per_10ms = (sample_rate / 100);
258 DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
260 source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
261 sink_params_.channel_layout(), sink_params_.channels(),
262 sample_rate, 16, frames_per_10ms);
264 const int frames_per_buffer =
265 GetOptimalBufferSize(sample_rate, sink_params_.frames_per_buffer());
267 sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
268 sink_params_.channels(), sample_rate, 16,
269 frames_per_buffer);
271 // Create a FIFO if re-buffering is required to match the source input with
272 // the sink request. The source acts as provider here and the sink as
273 // consumer.
274 fifo_delay_milliseconds_ = 0;
275 if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
276 DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
277 << " to " << sink_params_.frames_per_buffer();
278 audio_fifo_.reset(new media::AudioPullFifo(
279 source_params.channels(),
280 source_params.frames_per_buffer(),
281 base::Bind(
282 &WebRtcAudioRenderer::SourceCallback,
283 base::Unretained(this))));
285 if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
286 int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
287 static_cast<double>(source_params.sample_rate());
288 fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
289 source_params.frames_per_buffer()) * frame_duration_milliseconds;
293 source_ = source;
295 // Configure the audio rendering client and start rendering.
296 sink_ = AudioDeviceFactory::NewOutputDevice(source_render_frame_id_);
298 DCHECK_GE(session_id_, 0);
299 sink_->InitializeWithSessionId(sink_params_, this, session_id_);
301 sink_->Start();
303 // User must call Play() before any audio can be heard.
304 state_ = PAUSED;
306 return true;
309 scoped_refptr<MediaStreamAudioRenderer>
310 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
311 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
312 content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
313 base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
314 return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
317 bool WebRtcAudioRenderer::IsStarted() const {
318 DCHECK(thread_checker_.CalledOnValidThread());
319 return start_ref_count_ != 0;
322 void WebRtcAudioRenderer::Start() {
323 DVLOG(1) << "WebRtcAudioRenderer::Start()";
324 DCHECK(thread_checker_.CalledOnValidThread());
325 ++start_ref_count_;
328 void WebRtcAudioRenderer::Play() {
329 DVLOG(1) << "WebRtcAudioRenderer::Play()";
330 DCHECK(thread_checker_.CalledOnValidThread());
332 if (playing_state_.playing())
333 return;
335 playing_state_.set_playing(true);
336 render_callback_count_ = 0;
338 OnPlayStateChanged(media_stream_, &playing_state_);
341 void WebRtcAudioRenderer::EnterPlayState() {
342 DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
343 DCHECK(thread_checker_.CalledOnValidThread());
344 DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
345 base::AutoLock auto_lock(lock_);
346 if (state_ == UNINITIALIZED)
347 return;
349 DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
350 ++play_ref_count_;
352 if (state_ != PLAYING) {
353 state_ = PLAYING;
355 if (audio_fifo_) {
356 audio_delay_milliseconds_ = 0;
357 audio_fifo_->Clear();
362 void WebRtcAudioRenderer::Pause() {
363 DVLOG(1) << "WebRtcAudioRenderer::Pause()";
364 DCHECK(thread_checker_.CalledOnValidThread());
365 if (!playing_state_.playing())
366 return;
368 playing_state_.set_playing(false);
370 OnPlayStateChanged(media_stream_, &playing_state_);
373 void WebRtcAudioRenderer::EnterPauseState() {
374 DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
375 DCHECK(thread_checker_.CalledOnValidThread());
376 DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
377 base::AutoLock auto_lock(lock_);
378 if (state_ == UNINITIALIZED)
379 return;
381 DCHECK_EQ(state_, PLAYING);
382 DCHECK_GT(play_ref_count_, 0);
383 if (!--play_ref_count_)
384 state_ = PAUSED;
387 void WebRtcAudioRenderer::Stop() {
388 DVLOG(1) << "WebRtcAudioRenderer::Stop()";
389 DCHECK(thread_checker_.CalledOnValidThread());
391 base::AutoLock auto_lock(lock_);
392 if (state_ == UNINITIALIZED)
393 return;
395 if (--start_ref_count_)
396 return;
398 DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
400 source_->RemoveAudioRenderer(this);
401 source_ = NULL;
402 state_ = UNINITIALIZED;
405 // Make sure to stop the sink while _not_ holding the lock since the Render()
406 // callback may currently be executing and try to grab the lock while we're
407 // stopping the thread on which it runs.
408 sink_->Stop();
411 void WebRtcAudioRenderer::SetVolume(float volume) {
412 DCHECK(thread_checker_.CalledOnValidThread());
413 DCHECK(volume >= 0.0f && volume <= 1.0f);
415 playing_state_.set_volume(volume);
416 OnPlayStateChanged(media_stream_, &playing_state_);
419 void WebRtcAudioRenderer::SwitchOutputDevice(
420 const std::string& device_id,
421 const GURL& security_origin,
422 const media::SwitchOutputDeviceCB& callback) {
423 DCHECK(thread_checker_.CalledOnValidThread());
424 DCHECK(sink_);
425 DVLOG(1) << __FUNCTION__
426 << "(" << device_id << ", " << security_origin << ")";
427 sink_->SwitchOutputDevice(device_id, security_origin, callback);
430 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
431 DCHECK(thread_checker_.CalledOnValidThread());
432 base::AutoLock auto_lock(lock_);
433 return current_time_;
436 bool WebRtcAudioRenderer::IsLocalRenderer() const {
437 return false;
440 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
441 int audio_delay_milliseconds) {
442 base::AutoLock auto_lock(lock_);
443 if (!source_)
444 return 0;
446 DVLOG(2) << "WebRtcAudioRenderer::Render()";
447 DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
449 audio_delay_milliseconds_ = audio_delay_milliseconds;
451 if (audio_fifo_)
452 audio_fifo_->Consume(audio_bus, audio_bus->frames());
453 else
454 SourceCallback(0, audio_bus);
456 return (state_ == PLAYING) ? audio_bus->frames() : 0;
459 void WebRtcAudioRenderer::OnRenderError() {
460 NOTIMPLEMENTED();
461 LOG(ERROR) << "OnRenderError()";
464 // Called by AudioPullFifo when more data is necessary.
465 void WebRtcAudioRenderer::SourceCallback(
466 int fifo_frame_delay, media::AudioBus* audio_bus) {
467 base::TimeTicks start_time = base::TimeTicks::Now() ;
468 DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
469 << fifo_frame_delay << ", "
470 << audio_bus->frames() << ")";
472 int output_delay_milliseconds = audio_delay_milliseconds_;
473 output_delay_milliseconds += fifo_delay_milliseconds_;
474 DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
476 // We need to keep render data for the |source_| regardless of |state_|,
477 // otherwise the data will be buffered up inside |source_|.
478 source_->RenderData(audio_bus, sink_params_.sample_rate(),
479 output_delay_milliseconds,
480 &current_time_);
482 // Avoid filling up the audio bus if we are not playing; instead
483 // return here and ensure that the returned value in Render() is 0.
484 if (state_ != PLAYING)
485 audio_bus->Zero();
487 if (++render_callback_count_ == kNumCallbacksBetweenRenderTimeHistograms) {
488 base::TimeDelta elapsed = base::TimeTicks::Now() - start_time;
489 render_callback_count_ = 0;
490 UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed);
494 void WebRtcAudioRenderer::UpdateSourceVolume(
495 webrtc::AudioSourceInterface* source) {
496 DCHECK(thread_checker_.CalledOnValidThread());
498 // Note: If there are no playing audio renderers, then the volume will be
499 // set to 0.0.
500 float volume = 0.0f;
502 SourcePlayingStates::iterator entry = source_playing_states_.find(source);
503 if (entry != source_playing_states_.end()) {
504 PlayingStates& states = entry->second;
505 for (PlayingStates::const_iterator it = states.begin();
506 it != states.end(); ++it) {
507 if ((*it)->playing())
508 volume += (*it)->volume();
512 // The valid range for volume scaling of a remote webrtc source is
513 // 0.0-10.0 where 1.0 is no attenuation/boost.
514 DCHECK(volume >= 0.0f);
515 if (volume > 10.0f)
516 volume = 10.0f;
518 DVLOG(1) << "Setting remote source volume: " << volume;
519 if (!signaling_thread_->BelongsToCurrentThread()) {
520 // Libjingle hands out proxy objects in most cases, but the audio source
521 // object is an exception (bug?). So, to work around that, we need to make
522 // sure we call SetVolume on the signaling thread.
523 signaling_thread_->PostTask(FROM_HERE,
524 base::Bind(&webrtc::AudioSourceInterface::SetVolume, source, volume));
525 } else {
526 source->SetVolume(volume);
530 bool WebRtcAudioRenderer::AddPlayingState(
531 webrtc::AudioSourceInterface* source,
532 PlayingState* state) {
533 DCHECK(thread_checker_.CalledOnValidThread());
534 DCHECK(state->playing());
535 // Look up or add the |source| to the map.
536 PlayingStates& array = source_playing_states_[source];
537 if (std::find(array.begin(), array.end(), state) != array.end())
538 return false;
540 array.push_back(state);
542 return true;
545 bool WebRtcAudioRenderer::RemovePlayingState(
546 webrtc::AudioSourceInterface* source,
547 PlayingState* state) {
548 DCHECK(thread_checker_.CalledOnValidThread());
549 DCHECK(!state->playing());
550 SourcePlayingStates::iterator found = source_playing_states_.find(source);
551 if (found == source_playing_states_.end())
552 return false;
554 PlayingStates& array = found->second;
555 PlayingStates::iterator state_it =
556 std::find(array.begin(), array.end(), state);
557 if (state_it == array.end())
558 return false;
560 array.erase(state_it);
562 if (array.empty())
563 source_playing_states_.erase(found);
565 return true;
568 void WebRtcAudioRenderer::OnPlayStateChanged(
569 const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
570 PlayingState* state) {
571 webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
572 for (webrtc::AudioTrackVector::iterator it = tracks.begin();
573 it != tracks.end(); ++it) {
574 webrtc::AudioSourceInterface* source = (*it)->GetSource();
575 DCHECK(source);
576 if (!state->playing()) {
577 if (RemovePlayingState(source, state))
578 EnterPauseState();
579 } else if (AddPlayingState(source, state)) {
580 EnterPlayState();
582 UpdateSourceVolume(source);
586 } // namespace content