content/renderer/media/webrtc_audio_renderer.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_renderer.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/strings/stringprintf.h"
  11 #include "content/renderer/media/audio_device_factory.h"
  12 #include "content/renderer/media/media_stream_dispatcher.h"
  13 #include "content/renderer/media/webrtc_audio_device_impl.h"
  14 #include "content/renderer/media/webrtc_logging.h"
  15 #include "content/renderer/render_frame_impl.h"
  16 #include "media/audio/audio_output_device.h"
  17 #include "media/audio/audio_parameters.h"
  18 #include "media/audio/sample_rates.h"
  19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
  20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
  21
  22
  23 #if defined(OS_WIN)
  24 #include "base/win/windows_version.h"
  25 #include "media/audio/win/core_audio_util_win.h"
  26 #endif
  27
  28 namespace content {
  29
  30 namespace {
  31
  32 // We add a UMA histogram measuring the execution time of the Render() method
  33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
  34 // between each callback leads to one UMA update each 100ms.
  35 const int kNumCallbacksBetweenRenderTimeHistograms = 10;
  36
  37 // This is a simple wrapper class that's handed out to users of a shared
  38 // WebRtcAudioRenderer instance.  This class maintains the per-user 'playing'
  39 // and 'started' states to avoid problems related to incorrect usage which
  40 // might violate the implementation assumptions inside WebRtcAudioRenderer
  41 // (see the play reference count).
  42 class SharedAudioRenderer : public MediaStreamAudioRenderer {
  43  public:
  44   // Callback definition for a callback that is called when when Play(), Pause()
  45   // or SetVolume are called (whenever the internal |playing_state_| changes).
  46   typedef base::Callback<
  47       void(const scoped_refptr<webrtc::MediaStreamInterface>&,
  48            WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
  49
  50   SharedAudioRenderer(
  51       const scoped_refptr<MediaStreamAudioRenderer>& delegate,
  52       const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
  53       const OnPlayStateChanged& on_play_state_changed)
  54       : delegate_(delegate), media_stream_(media_stream), started_(false),
  55         on_play_state_changed_(on_play_state_changed) {
  56     DCHECK(!on_play_state_changed_.is_null());
  57     DCHECK(media_stream_.get());
  58   }
  59
  60  protected:
  61   ~SharedAudioRenderer() override {
  62     DCHECK(thread_checker_.CalledOnValidThread());
  63     DVLOG(1) << __FUNCTION__;
  64     Stop();
  65   }
  66
  67   void Start() override {
  68     DCHECK(thread_checker_.CalledOnValidThread());
  69     if (started_)
  70       return;
  71     started_ = true;
  72     delegate_->Start();
  73   }
  74
  75   void Play() override {
  76     DCHECK(thread_checker_.CalledOnValidThread());
  77     DCHECK(started_);
  78     if (playing_state_.playing())
  79       return;
  80     playing_state_.set_playing(true);
  81     on_play_state_changed_.Run(media_stream_, &playing_state_);
  82   }
  83
  84   void Pause() override {
  85     DCHECK(thread_checker_.CalledOnValidThread());
  86     DCHECK(started_);
  87     if (!playing_state_.playing())
  88       return;
  89     playing_state_.set_playing(false);
  90     on_play_state_changed_.Run(media_stream_, &playing_state_);
  91   }
  92
  93   void Stop() override {
  94     DCHECK(thread_checker_.CalledOnValidThread());
  95     if (!started_)
  96       return;
  97     Pause();
  98     started_ = false;
  99     delegate_->Stop();
 100   }
 101
 102   void SetVolume(float volume) override {
 103     DCHECK(thread_checker_.CalledOnValidThread());
 104     DCHECK(volume >= 0.0f && volume <= 1.0f);
 105     playing_state_.set_volume(volume);
 106     on_play_state_changed_.Run(media_stream_, &playing_state_);
 107   }
 108
 109   base::TimeDelta GetCurrentRenderTime() const override {
 110     DCHECK(thread_checker_.CalledOnValidThread());
 111     return delegate_->GetCurrentRenderTime();
 112   }
 113
 114   bool IsLocalRenderer() const override {
 115     DCHECK(thread_checker_.CalledOnValidThread());
 116     return delegate_->IsLocalRenderer();
 117   }
 118
 119  private:
 120   base::ThreadChecker thread_checker_;
 121   const scoped_refptr<MediaStreamAudioRenderer> delegate_;
 122   const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
 123   bool started_;
 124   WebRtcAudioRenderer::PlayingState playing_state_;
 125   OnPlayStateChanged on_play_state_changed_;
 126 };
 127
 128 // Returns either AudioParameters::NO_EFFECTS or AudioParameters::DUCKING
 129 // depending on whether or not an input element is currently open with
 130 // ducking enabled.
 131 int GetCurrentDuckingFlag(int render_frame_id) {
 132   RenderFrameImpl* const frame =
 133       RenderFrameImpl::FromRoutingID(render_frame_id);
 134   MediaStreamDispatcher* const dispatcher = frame ?
 135       frame->GetMediaStreamDispatcher() : NULL;
 136   if (dispatcher && dispatcher->IsAudioDuckingActive()) {
 137     return media::AudioParameters::DUCKING;
 138   }
 139
 140   return media::AudioParameters::NO_EFFECTS;
 141 }
 142
 143 }  // namespace
 144
 145 int WebRtcAudioRenderer::GetOptimalBufferSize(int sample_rate,
 146                                               int hardware_buffer_size) {
 147   // Use native hardware buffer size as default. On Windows, we strive to open
 148   // up using this native hardware buffer size to achieve best
 149   // possible performance and to ensure that no FIFO is needed on the browser
 150   // side to match the client request. That is why there is no #if case for
 151   // Windows below.
 152   int frames_per_buffer = hardware_buffer_size;
 153
 154 #if defined(OS_LINUX) || defined(OS_MACOSX)
 155   // On Linux and MacOS, the low level IO implementations on the browser side
 156   // supports all buffer size the clients want. We use the native peer
 157   // connection buffer size (10ms) to achieve best possible performance.
 158   frames_per_buffer = sample_rate / 100;
 159 #elif defined(OS_ANDROID)
 160   // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
 161   // cases. Might not be possible to come up with the perfect solution using
 162   // the render side only.
 163   int frames_per_10ms = sample_rate / 100;
 164   if (frames_per_buffer < 2 * frames_per_10ms) {
 165     // Examples of low-latency frame sizes and the resulting |buffer_size|:
 166     //  Nexus 7     : 240 audio frames => 2*480 = 960
 167     //  Nexus 10    : 256              => 2*441 = 882
 168     //  Galaxy Nexus: 144              => 2*441 = 882
 169     frames_per_buffer = 2 * frames_per_10ms;
 170     DVLOG(1) << "Low-latency output detected on Android";
 171   }
 172 #endif
 173
 174   DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
 175   return frames_per_buffer;
 176 }
 177
 178 WebRtcAudioRenderer::WebRtcAudioRenderer(
 179     const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread,
 180     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
 181     int source_render_view_id,
 182     int source_render_frame_id,
 183     int session_id,
 184     int sample_rate,
 185     int frames_per_buffer)
 186     : state_(UNINITIALIZED),
 187       source_render_view_id_(source_render_view_id),
 188       source_render_frame_id_(source_render_frame_id),
 189       session_id_(session_id),
 190       signaling_thread_(signaling_thread),
 191       media_stream_(media_stream),
 192       source_(NULL),
 193       play_ref_count_(0),
 194       start_ref_count_(0),
 195       audio_delay_milliseconds_(0),
 196       fifo_delay_milliseconds_(0),
 197       sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 198                    media::CHANNEL_LAYOUT_STEREO, sample_rate, 16,
 199                    frames_per_buffer,
 200                    GetCurrentDuckingFlag(source_render_frame_id)),
 201       render_callback_count_(0) {
 202   WebRtcLogMessage(base::StringPrintf(
 203       "WAR::WAR. source_render_view_id=%d"
 204       ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
 205       source_render_view_id,
 206       session_id,
 207       sample_rate,
 208       frames_per_buffer,
 209       sink_params_.effects()));
 210 }
 211
 212 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
 213   DCHECK(thread_checker_.CalledOnValidThread());
 214   DCHECK_EQ(state_, UNINITIALIZED);
 215 }
 216
 217 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
 218   DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
 219   DCHECK(thread_checker_.CalledOnValidThread());
 220   base::AutoLock auto_lock(lock_);
 221   DCHECK_EQ(state_, UNINITIALIZED);
 222   DCHECK(source);
 223   DCHECK(!sink_.get());
 224   DCHECK(!source_);
 225
 226   // WebRTC does not yet support higher rates than 96000 on the client side
 227   // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
 228   // we change the rate to 48000 instead. The consequence is that the native
 229   // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
 230   // which will then be resampled by the audio converted on the browser side
 231   // to match the native audio layer.
 232   int sample_rate = sink_params_.sample_rate();
 233   DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
 234   if (sample_rate == 192000) {
 235     DVLOG(1) << "Resampling from 48000 to 192000 is required";
 236     sample_rate = 48000;
 237   }
 238   media::AudioSampleRate asr;
 239   if (media::ToAudioSampleRate(sample_rate, &asr)) {
 240     UMA_HISTOGRAM_ENUMERATION(
 241         "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
 242   } else {
 243     UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
 244                          sample_rate);
 245   }
 246
 247   // Set up audio parameters for the source, i.e., the WebRTC client.
 248
 249   // The WebRTC client only supports multiples of 10ms as buffer size where
 250   // 10ms is preferred for lowest possible delay.
 251   media::AudioParameters source_params;
 252   const int frames_per_10ms = (sample_rate / 100);
 253   DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
 254
 255   source_params.Reset(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 256                       sink_params_.channel_layout(), sink_params_.channels(),
 257                       sample_rate, 16, frames_per_10ms);
 258
 259   const int frames_per_buffer =
 260       GetOptimalBufferSize(sample_rate, sink_params_.frames_per_buffer());
 261
 262   sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
 263                      sink_params_.channels(), sample_rate, 16,
 264                      frames_per_buffer);
 265
 266   // Create a FIFO if re-buffering is required to match the source input with
 267   // the sink request. The source acts as provider here and the sink as
 268   // consumer.
 269   fifo_delay_milliseconds_ = 0;
 270   if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
 271     DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
 272              << " to " << sink_params_.frames_per_buffer();
 273     audio_fifo_.reset(new media::AudioPullFifo(
 274         source_params.channels(),
 275         source_params.frames_per_buffer(),
 276         base::Bind(
 277             &WebRtcAudioRenderer::SourceCallback,
 278             base::Unretained(this))));
 279
 280     if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
 281       int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
 282           static_cast<double>(source_params.sample_rate());
 283       fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
 284         source_params.frames_per_buffer()) * frame_duration_milliseconds;
 285     }
 286   }
 287
 288   source_ = source;
 289
 290   // Configure the audio rendering client and start rendering.
 291   sink_ = AudioDeviceFactory::NewOutputDevice(
 292       source_render_view_id_, source_render_frame_id_);
 293
 294   DCHECK_GE(session_id_, 0);
 295   sink_->InitializeWithSessionId(sink_params_, this, session_id_);
 296
 297   sink_->Start();
 298
 299   // User must call Play() before any audio can be heard.
 300   state_ = PAUSED;
 301
 302   return true;
 303 }
 304
 305 scoped_refptr<MediaStreamAudioRenderer>
 306 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
 307     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
 308   content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
 309       base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
 310   return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
 311 }
 312
 313 bool WebRtcAudioRenderer::IsStarted() const {
 314   DCHECK(thread_checker_.CalledOnValidThread());
 315   return start_ref_count_ != 0;
 316 }
 317
 318 void WebRtcAudioRenderer::Start() {
 319   DVLOG(1) << "WebRtcAudioRenderer::Start()";
 320   DCHECK(thread_checker_.CalledOnValidThread());
 321   ++start_ref_count_;
 322 }
 323
 324 void WebRtcAudioRenderer::Play() {
 325   DVLOG(1) << "WebRtcAudioRenderer::Play()";
 326   DCHECK(thread_checker_.CalledOnValidThread());
 327
 328   if (playing_state_.playing())
 329     return;
 330
 331   playing_state_.set_playing(true);
 332   render_callback_count_ = 0;
 333
 334   OnPlayStateChanged(media_stream_, &playing_state_);
 335 }
 336
 337 void WebRtcAudioRenderer::EnterPlayState() {
 338   DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
 339   DCHECK(thread_checker_.CalledOnValidThread());
 340   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
 341   base::AutoLock auto_lock(lock_);
 342   if (state_ == UNINITIALIZED)
 343     return;
 344
 345   DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
 346   ++play_ref_count_;
 347
 348   if (state_ != PLAYING) {
 349     state_ = PLAYING;
 350
 351     if (audio_fifo_) {
 352       audio_delay_milliseconds_ = 0;
 353       audio_fifo_->Clear();
 354     }
 355   }
 356 }
 357
 358 void WebRtcAudioRenderer::Pause() {
 359   DVLOG(1) << "WebRtcAudioRenderer::Pause()";
 360   DCHECK(thread_checker_.CalledOnValidThread());
 361   if (!playing_state_.playing())
 362     return;
 363
 364   playing_state_.set_playing(false);
 365
 366   OnPlayStateChanged(media_stream_, &playing_state_);
 367 }
 368
 369 void WebRtcAudioRenderer::EnterPauseState() {
 370   DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
 371   DCHECK(thread_checker_.CalledOnValidThread());
 372   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
 373   base::AutoLock auto_lock(lock_);
 374   if (state_ == UNINITIALIZED)
 375     return;
 376
 377   DCHECK_EQ(state_, PLAYING);
 378   DCHECK_GT(play_ref_count_, 0);
 379   if (!--play_ref_count_)
 380     state_ = PAUSED;
 381 }
 382
 383 void WebRtcAudioRenderer::Stop() {
 384   DVLOG(1) << "WebRtcAudioRenderer::Stop()";
 385   DCHECK(thread_checker_.CalledOnValidThread());
 386   {
 387     base::AutoLock auto_lock(lock_);
 388     if (state_ == UNINITIALIZED)
 389       return;
 390
 391     if (--start_ref_count_)
 392       return;
 393
 394     DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
 395
 396     source_->RemoveAudioRenderer(this);
 397     source_ = NULL;
 398     state_ = UNINITIALIZED;
 399   }
 400
 401   // Make sure to stop the sink while _not_ holding the lock since the Render()
 402   // callback may currently be executing and try to grab the lock while we're
 403   // stopping the thread on which it runs.
 404   sink_->Stop();
 405 }
 406
 407 void WebRtcAudioRenderer::SetVolume(float volume) {
 408   DCHECK(thread_checker_.CalledOnValidThread());
 409   DCHECK(volume >= 0.0f && volume <= 1.0f);
 410
 411   playing_state_.set_volume(volume);
 412   OnPlayStateChanged(media_stream_, &playing_state_);
 413 }
 414
 415 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
 416   DCHECK(thread_checker_.CalledOnValidThread());
 417   base::AutoLock auto_lock(lock_);
 418   return current_time_;
 419 }
 420
 421 bool WebRtcAudioRenderer::IsLocalRenderer() const {
 422   return false;
 423 }
 424
 425 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
 426                                 int audio_delay_milliseconds) {
 427   base::AutoLock auto_lock(lock_);
 428   if (!source_)
 429     return 0;
 430
 431   DVLOG(2) << "WebRtcAudioRenderer::Render()";
 432   DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
 433
 434   audio_delay_milliseconds_ = audio_delay_milliseconds;
 435
 436   if (audio_fifo_)
 437     audio_fifo_->Consume(audio_bus, audio_bus->frames());
 438   else
 439     SourceCallback(0, audio_bus);
 440
 441   return (state_ == PLAYING) ? audio_bus->frames() : 0;
 442 }
 443
 444 void WebRtcAudioRenderer::OnRenderError() {
 445   NOTIMPLEMENTED();
 446   LOG(ERROR) << "OnRenderError()";
 447 }
 448
 449 // Called by AudioPullFifo when more data is necessary.
 450 void WebRtcAudioRenderer::SourceCallback(
 451     int fifo_frame_delay, media::AudioBus* audio_bus) {
 452   base::TimeTicks start_time = base::TimeTicks::Now() ;
 453   DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
 454            << fifo_frame_delay << ", "
 455            << audio_bus->frames() << ")";
 456
 457   int output_delay_milliseconds = audio_delay_milliseconds_;
 458   output_delay_milliseconds += fifo_delay_milliseconds_;
 459   DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
 460
 461   // We need to keep render data for the |source_| regardless of |state_|,
 462   // otherwise the data will be buffered up inside |source_|.
 463   source_->RenderData(audio_bus, sink_params_.sample_rate(),
 464                       output_delay_milliseconds,
 465                       &current_time_);
 466
 467   // Avoid filling up the audio bus if we are not playing; instead
 468   // return here and ensure that the returned value in Render() is 0.
 469   if (state_ != PLAYING)
 470     audio_bus->Zero();
 471
 472   if (++render_callback_count_ == kNumCallbacksBetweenRenderTimeHistograms) {
 473     base::TimeDelta elapsed = base::TimeTicks::Now() - start_time;
 474     render_callback_count_ = 0;
 475     UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed);
 476   }
 477 }
 478
 479 void WebRtcAudioRenderer::UpdateSourceVolume(
 480     webrtc::AudioSourceInterface* source) {
 481   DCHECK(thread_checker_.CalledOnValidThread());
 482
 483   // Note: If there are no playing audio renderers, then the volume will be
 484   // set to 0.0.
 485   float volume = 0.0f;
 486
 487   SourcePlayingStates::iterator entry = source_playing_states_.find(source);
 488   if (entry != source_playing_states_.end()) {
 489     PlayingStates& states = entry->second;
 490     for (PlayingStates::const_iterator it = states.begin();
 491          it != states.end(); ++it) {
 492       if ((*it)->playing())
 493         volume += (*it)->volume();
 494     }
 495   }
 496
 497   // The valid range for volume scaling of a remote webrtc source is
 498   // 0.0-10.0 where 1.0 is no attenuation/boost.
 499   DCHECK(volume >= 0.0f);
 500   if (volume > 10.0f)
 501     volume = 10.0f;
 502
 503   DVLOG(1) << "Setting remote source volume: " << volume;
 504   if (!signaling_thread_->BelongsToCurrentThread()) {
 505     // Libjingle hands out proxy objects in most cases, but the audio source
 506     // object is an exception (bug?).  So, to work around that, we need to make
 507     // sure we call SetVolume on the signaling thread.
 508     signaling_thread_->PostTask(FROM_HERE,
 509         base::Bind(&webrtc::AudioSourceInterface::SetVolume, source, volume));
 510   } else {
 511     source->SetVolume(volume);
 512   }
 513 }
 514
 515 bool WebRtcAudioRenderer::AddPlayingState(
 516     webrtc::AudioSourceInterface* source,
 517     PlayingState* state) {
 518   DCHECK(thread_checker_.CalledOnValidThread());
 519   DCHECK(state->playing());
 520   // Look up or add the |source| to the map.
 521   PlayingStates& array = source_playing_states_[source];
 522   if (std::find(array.begin(), array.end(), state) != array.end())
 523     return false;
 524
 525   array.push_back(state);
 526
 527   return true;
 528 }
 529
 530 bool WebRtcAudioRenderer::RemovePlayingState(
 531     webrtc::AudioSourceInterface* source,
 532     PlayingState* state) {
 533   DCHECK(thread_checker_.CalledOnValidThread());
 534   DCHECK(!state->playing());
 535   SourcePlayingStates::iterator found = source_playing_states_.find(source);
 536   if (found == source_playing_states_.end())
 537     return false;
 538
 539   PlayingStates& array = found->second;
 540   PlayingStates::iterator state_it =
 541       std::find(array.begin(), array.end(), state);
 542   if (state_it == array.end())
 543     return false;
 544
 545   array.erase(state_it);
 546
 547   if (array.empty())
 548     source_playing_states_.erase(found);
 549
 550   return true;
 551 }
 552
 553 void WebRtcAudioRenderer::OnPlayStateChanged(
 554     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
 555     PlayingState* state) {
 556   webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
 557   for (webrtc::AudioTrackVector::iterator it = tracks.begin();
 558        it != tracks.end(); ++it) {
 559     webrtc::AudioSourceInterface* source = (*it)->GetSource();
 560     DCHECK(source);
 561     if (!state->playing()) {
 562       if (RemovePlayingState(source, state))
 563         EnterPauseState();
 564     } else if (AddPlayingState(source, state)) {
 565       EnterPlayState();
 566     }
 567     UpdateSourceVolume(source);
 568   }
 569 }
 570
 571 }  // namespace content