content/renderer/media/webrtc_audio_renderer.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_renderer.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/strings/stringprintf.h"
  11 #include "content/renderer/media/audio_device_factory.h"
  12 #include "content/renderer/media/media_stream_dispatcher.h"
  13 #include "content/renderer/media/webrtc_audio_device_impl.h"
  14 #include "content/renderer/media/webrtc_logging.h"
  15 #include "content/renderer/render_frame_impl.h"
  16 #include "media/audio/audio_output_device.h"
  17 #include "media/audio/audio_parameters.h"
  18 #include "media/audio/sample_rates.h"
  19 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
  20 #include "third_party/libjingle/source/talk/media/base/audiorenderer.h"
  21
  22
  23 #if defined(OS_WIN)
  24 #include "base/win/windows_version.h"
  25 #include "media/audio/win/core_audio_util_win.h"
  26 #endif
  27
  28 namespace content {
  29
  30 namespace {
  31
  32 // We add a UMA histogram measuring the execution time of the Render() method
  33 // every |kNumCallbacksBetweenRenderTimeHistograms| callback. Assuming 10ms
  34 // between each callback leads to one UMA update each 100ms.
  35 const int kNumCallbacksBetweenRenderTimeHistograms = 10;
  36
  37 // This is a simple wrapper class that's handed out to users of a shared
  38 // WebRtcAudioRenderer instance.  This class maintains the per-user 'playing'
  39 // and 'started' states to avoid problems related to incorrect usage which
  40 // might violate the implementation assumptions inside WebRtcAudioRenderer
  41 // (see the play reference count).
  42 class SharedAudioRenderer : public MediaStreamAudioRenderer {
  43  public:
  44   // Callback definition for a callback that is called when when Play(), Pause()
  45   // or SetVolume are called (whenever the internal |playing_state_| changes).
  46   typedef base::Callback<
  47       void(const scoped_refptr<webrtc::MediaStreamInterface>&,
  48            WebRtcAudioRenderer::PlayingState*)> OnPlayStateChanged;
  49
  50   SharedAudioRenderer(
  51       const scoped_refptr<MediaStreamAudioRenderer>& delegate,
  52       const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
  53       const OnPlayStateChanged& on_play_state_changed)
  54       : delegate_(delegate), media_stream_(media_stream), started_(false),
  55         on_play_state_changed_(on_play_state_changed) {
  56     DCHECK(!on_play_state_changed_.is_null());
  57     DCHECK(media_stream_.get());
  58   }
  59
  60  protected:
  61   ~SharedAudioRenderer() override {
  62     DCHECK(thread_checker_.CalledOnValidThread());
  63     DVLOG(1) << __FUNCTION__;
  64     Stop();
  65   }
  66
  67   void Start() override {
  68     DCHECK(thread_checker_.CalledOnValidThread());
  69     if (started_)
  70       return;
  71     started_ = true;
  72     delegate_->Start();
  73   }
  74
  75   void Play() override {
  76     DCHECK(thread_checker_.CalledOnValidThread());
  77     DCHECK(started_);
  78     if (playing_state_.playing())
  79       return;
  80     playing_state_.set_playing(true);
  81     on_play_state_changed_.Run(media_stream_, &playing_state_);
  82   }
  83
  84   void Pause() override {
  85     DCHECK(thread_checker_.CalledOnValidThread());
  86     DCHECK(started_);
  87     if (!playing_state_.playing())
  88       return;
  89     playing_state_.set_playing(false);
  90     on_play_state_changed_.Run(media_stream_, &playing_state_);
  91   }
  92
  93   void Stop() override {
  94     DCHECK(thread_checker_.CalledOnValidThread());
  95     if (!started_)
  96       return;
  97     Pause();
  98     started_ = false;
  99     delegate_->Stop();
 100   }
 101
 102   void SetVolume(float volume) override {
 103     DCHECK(thread_checker_.CalledOnValidThread());
 104     DCHECK(volume >= 0.0f && volume <= 1.0f);
 105     playing_state_.set_volume(volume);
 106     on_play_state_changed_.Run(media_stream_, &playing_state_);
 107   }
 108
 109   media::OutputDevice* GetOutputDevice() override {
 110     DVLOG(1) << __FUNCTION__;
 111     DCHECK(thread_checker_.CalledOnValidThread());
 112     return delegate_->GetOutputDevice();
 113   }
 114
 115   base::TimeDelta GetCurrentRenderTime() const override {
 116     DCHECK(thread_checker_.CalledOnValidThread());
 117     return delegate_->GetCurrentRenderTime();
 118   }
 119
 120   bool IsLocalRenderer() const override {
 121     DCHECK(thread_checker_.CalledOnValidThread());
 122     return delegate_->IsLocalRenderer();
 123   }
 124
 125  private:
 126   base::ThreadChecker thread_checker_;
 127   const scoped_refptr<MediaStreamAudioRenderer> delegate_;
 128   const scoped_refptr<webrtc::MediaStreamInterface> media_stream_;
 129   bool started_;
 130   WebRtcAudioRenderer::PlayingState playing_state_;
 131   OnPlayStateChanged on_play_state_changed_;
 132 };
 133
 134 }  // namespace
 135
 136 int WebRtcAudioRenderer::GetOptimalBufferSize(int sample_rate,
 137                                               int hardware_buffer_size) {
 138   // Use native hardware buffer size as default. On Windows, we strive to open
 139   // up using this native hardware buffer size to achieve best
 140   // possible performance and to ensure that no FIFO is needed on the browser
 141   // side to match the client request. That is why there is no #if case for
 142   // Windows below.
 143   int frames_per_buffer = hardware_buffer_size;
 144
 145 #if defined(OS_LINUX) || defined(OS_MACOSX)
 146   // On Linux and MacOS, the low level IO implementations on the browser side
 147   // supports all buffer size the clients want. We use the native peer
 148   // connection buffer size (10ms) to achieve best possible performance.
 149   frames_per_buffer = sample_rate / 100;
 150 #elif defined(OS_ANDROID)
 151   // TODO(henrika): Keep tuning this scheme and espcicially for low-latency
 152   // cases. Might not be possible to come up with the perfect solution using
 153   // the render side only.
 154   int frames_per_10ms = sample_rate / 100;
 155   if (frames_per_buffer < 2 * frames_per_10ms) {
 156     // Examples of low-latency frame sizes and the resulting |buffer_size|:
 157     //  Nexus 7     : 240 audio frames => 2*480 = 960
 158     //  Nexus 10    : 256              => 2*441 = 882
 159     //  Galaxy Nexus: 144              => 2*441 = 882
 160     frames_per_buffer = 2 * frames_per_10ms;
 161     DVLOG(1) << "Low-latency output detected on Android";
 162   }
 163 #endif
 164
 165   DVLOG(1) << "Using sink output buffer size: " << frames_per_buffer;
 166   return frames_per_buffer;
 167 }
 168
 169 WebRtcAudioRenderer::WebRtcAudioRenderer(
 170     const scoped_refptr<base::SingleThreadTaskRunner>& signaling_thread,
 171     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
 172     int source_render_frame_id,
 173     int session_id,
 174     int sample_rate,
 175     int frames_per_buffer)
 176     : state_(UNINITIALIZED),
 177       source_render_frame_id_(source_render_frame_id),
 178       session_id_(session_id),
 179       signaling_thread_(signaling_thread),
 180       media_stream_(media_stream),
 181       source_(NULL),
 182       play_ref_count_(0),
 183       start_ref_count_(0),
 184       audio_delay_milliseconds_(0),
 185       fifo_delay_milliseconds_(0),
 186       sink_params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 187                    media::CHANNEL_LAYOUT_STEREO, sample_rate, 16,
 188                    frames_per_buffer),
 189       render_callback_count_(0) {
 190   WebRtcLogMessage(base::StringPrintf(
 191       "WAR::WAR. source_render_frame_id=%d"
 192       ", session_id=%d, sample_rate=%d, frames_per_buffer=%d, effects=%i",
 193       source_render_frame_id, session_id, sample_rate, frames_per_buffer,
 194       sink_params_.effects()));
 195 }
 196
 197 WebRtcAudioRenderer::~WebRtcAudioRenderer() {
 198   DCHECK(thread_checker_.CalledOnValidThread());
 199   DCHECK_EQ(state_, UNINITIALIZED);
 200 }
 201
 202 bool WebRtcAudioRenderer::Initialize(WebRtcAudioRendererSource* source) {
 203   DVLOG(1) << "WebRtcAudioRenderer::Initialize()";
 204   DCHECK(thread_checker_.CalledOnValidThread());
 205   base::AutoLock auto_lock(lock_);
 206   DCHECK_EQ(state_, UNINITIALIZED);
 207   DCHECK(source);
 208   DCHECK(!sink_.get());
 209   DCHECK(!source_);
 210
 211   // WebRTC does not yet support higher rates than 96000 on the client side
 212   // and 48000 is the preferred sample rate. Therefore, if 192000 is detected,
 213   // we change the rate to 48000 instead. The consequence is that the native
 214   // layer will be opened up at 192kHz but WebRTC will provide data at 48kHz
 215   // which will then be resampled by the audio converted on the browser side
 216   // to match the native audio layer.
 217   int sample_rate = sink_params_.sample_rate();
 218   DVLOG(1) << "Audio output hardware sample rate: " << sample_rate;
 219   if (sample_rate == 192000) {
 220     DVLOG(1) << "Resampling from 48000 to 192000 is required";
 221     sample_rate = 48000;
 222   }
 223   media::AudioSampleRate asr;
 224   if (media::ToAudioSampleRate(sample_rate, &asr)) {
 225     UMA_HISTOGRAM_ENUMERATION(
 226         "WebRTC.AudioOutputSampleRate", asr, media::kAudioSampleRateMax + 1);
 227   } else {
 228     UMA_HISTOGRAM_COUNTS("WebRTC.AudioOutputSampleRateUnexpected",
 229                          sample_rate);
 230   }
 231
 232   // Set up audio parameters for the source, i.e., the WebRTC client.
 233
 234   // The WebRTC client only supports multiples of 10ms as buffer size where
 235   // 10ms is preferred for lowest possible delay.
 236   const int frames_per_10ms = (sample_rate / 100);
 237   DVLOG(1) << "Using WebRTC output buffer size: " << frames_per_10ms;
 238   media::AudioParameters source_params(
 239       media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 240       sink_params_.channel_layout(), sample_rate, 16, frames_per_10ms);
 241   source_params.set_channels_for_discrete(sink_params_.channels());
 242
 243   const int frames_per_buffer =
 244       GetOptimalBufferSize(sample_rate, sink_params_.frames_per_buffer());
 245
 246   sink_params_.Reset(sink_params_.format(), sink_params_.channel_layout(),
 247                      sample_rate, 16, frames_per_buffer);
 248
 249   // Create a FIFO if re-buffering is required to match the source input with
 250   // the sink request. The source acts as provider here and the sink as
 251   // consumer.
 252   fifo_delay_milliseconds_ = 0;
 253   if (source_params.frames_per_buffer() != sink_params_.frames_per_buffer()) {
 254     DVLOG(1) << "Rebuffering from " << source_params.frames_per_buffer()
 255              << " to " << sink_params_.frames_per_buffer();
 256     audio_fifo_.reset(new media::AudioPullFifo(
 257         source_params.channels(),
 258         source_params.frames_per_buffer(),
 259         base::Bind(
 260             &WebRtcAudioRenderer::SourceCallback,
 261             base::Unretained(this))));
 262
 263     if (sink_params_.frames_per_buffer() > source_params.frames_per_buffer()) {
 264       int frame_duration_milliseconds = base::Time::kMillisecondsPerSecond /
 265           static_cast<double>(source_params.sample_rate());
 266       fifo_delay_milliseconds_ = (sink_params_.frames_per_buffer() -
 267         source_params.frames_per_buffer()) * frame_duration_milliseconds;
 268     }
 269   }
 270
 271   source_ = source;
 272
 273   // Configure the audio rendering client and start rendering.
 274   DCHECK_GE(session_id_, 0);
 275   sink_ = AudioDeviceFactory::NewOutputDevice(
 276       source_render_frame_id_, session_id_, std::string(), url::Origin());
 277   sink_->Initialize(sink_params_, this);
 278
 279   sink_->Start();
 280
 281   // User must call Play() before any audio can be heard.
 282   state_ = PAUSED;
 283
 284   return true;
 285 }
 286
 287 scoped_refptr<MediaStreamAudioRenderer>
 288 WebRtcAudioRenderer::CreateSharedAudioRendererProxy(
 289     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream) {
 290   content::SharedAudioRenderer::OnPlayStateChanged on_play_state_changed =
 291       base::Bind(&WebRtcAudioRenderer::OnPlayStateChanged, this);
 292   return new SharedAudioRenderer(this, media_stream, on_play_state_changed);
 293 }
 294
 295 bool WebRtcAudioRenderer::IsStarted() const {
 296   DCHECK(thread_checker_.CalledOnValidThread());
 297   return start_ref_count_ != 0;
 298 }
 299
 300 void WebRtcAudioRenderer::Start() {
 301   DVLOG(1) << "WebRtcAudioRenderer::Start()";
 302   DCHECK(thread_checker_.CalledOnValidThread());
 303   ++start_ref_count_;
 304 }
 305
 306 void WebRtcAudioRenderer::Play() {
 307   DVLOG(1) << "WebRtcAudioRenderer::Play()";
 308   DCHECK(thread_checker_.CalledOnValidThread());
 309
 310   if (playing_state_.playing())
 311     return;
 312
 313   playing_state_.set_playing(true);
 314   render_callback_count_ = 0;
 315
 316   OnPlayStateChanged(media_stream_, &playing_state_);
 317 }
 318
 319 void WebRtcAudioRenderer::EnterPlayState() {
 320   DVLOG(1) << "WebRtcAudioRenderer::EnterPlayState()";
 321   DCHECK(thread_checker_.CalledOnValidThread());
 322   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
 323   base::AutoLock auto_lock(lock_);
 324   if (state_ == UNINITIALIZED)
 325     return;
 326
 327   DCHECK(play_ref_count_ == 0 || state_ == PLAYING);
 328   ++play_ref_count_;
 329
 330   if (state_ != PLAYING) {
 331     state_ = PLAYING;
 332
 333     if (audio_fifo_) {
 334       audio_delay_milliseconds_ = 0;
 335       audio_fifo_->Clear();
 336     }
 337   }
 338 }
 339
 340 void WebRtcAudioRenderer::Pause() {
 341   DVLOG(1) << "WebRtcAudioRenderer::Pause()";
 342   DCHECK(thread_checker_.CalledOnValidThread());
 343   if (!playing_state_.playing())
 344     return;
 345
 346   playing_state_.set_playing(false);
 347
 348   OnPlayStateChanged(media_stream_, &playing_state_);
 349 }
 350
 351 void WebRtcAudioRenderer::EnterPauseState() {
 352   DVLOG(1) << "WebRtcAudioRenderer::EnterPauseState()";
 353   DCHECK(thread_checker_.CalledOnValidThread());
 354   DCHECK_GT(start_ref_count_, 0) << "Did you forget to call Start()?";
 355   base::AutoLock auto_lock(lock_);
 356   if (state_ == UNINITIALIZED)
 357     return;
 358
 359   DCHECK_EQ(state_, PLAYING);
 360   DCHECK_GT(play_ref_count_, 0);
 361   if (!--play_ref_count_)
 362     state_ = PAUSED;
 363 }
 364
 365 void WebRtcAudioRenderer::Stop() {
 366   DVLOG(1) << "WebRtcAudioRenderer::Stop()";
 367   DCHECK(thread_checker_.CalledOnValidThread());
 368   {
 369     base::AutoLock auto_lock(lock_);
 370     if (state_ == UNINITIALIZED)
 371       return;
 372
 373     if (--start_ref_count_)
 374       return;
 375
 376     DVLOG(1) << "Calling RemoveAudioRenderer and Stop().";
 377
 378     source_->RemoveAudioRenderer(this);
 379     source_ = NULL;
 380     state_ = UNINITIALIZED;
 381   }
 382
 383   // Make sure to stop the sink while _not_ holding the lock since the Render()
 384   // callback may currently be executing and try to grab the lock while we're
 385   // stopping the thread on which it runs.
 386   sink_->Stop();
 387 }
 388
 389 void WebRtcAudioRenderer::SetVolume(float volume) {
 390   DCHECK(thread_checker_.CalledOnValidThread());
 391   DCHECK(volume >= 0.0f && volume <= 1.0f);
 392
 393   playing_state_.set_volume(volume);
 394   OnPlayStateChanged(media_stream_, &playing_state_);
 395 }
 396
 397 media::OutputDevice* WebRtcAudioRenderer::GetOutputDevice() {
 398   DVLOG(1) << __FUNCTION__;
 399   DCHECK(thread_checker_.CalledOnValidThread());
 400   DCHECK(sink_);
 401   return sink_->GetOutputDevice();
 402 }
 403
 404 base::TimeDelta WebRtcAudioRenderer::GetCurrentRenderTime() const {
 405   DCHECK(thread_checker_.CalledOnValidThread());
 406   base::AutoLock auto_lock(lock_);
 407   return current_time_;
 408 }
 409
 410 bool WebRtcAudioRenderer::IsLocalRenderer() const {
 411   return false;
 412 }
 413
 414 int WebRtcAudioRenderer::Render(media::AudioBus* audio_bus,
 415                                 int audio_delay_milliseconds) {
 416   base::AutoLock auto_lock(lock_);
 417   if (!source_)
 418     return 0;
 419
 420   DVLOG(2) << "WebRtcAudioRenderer::Render()";
 421   DVLOG(2) << "audio_delay_milliseconds: " << audio_delay_milliseconds;
 422
 423   audio_delay_milliseconds_ = audio_delay_milliseconds;
 424
 425   if (audio_fifo_)
 426     audio_fifo_->Consume(audio_bus, audio_bus->frames());
 427   else
 428     SourceCallback(0, audio_bus);
 429
 430   return (state_ == PLAYING) ? audio_bus->frames() : 0;
 431 }
 432
 433 void WebRtcAudioRenderer::OnRenderError() {
 434   NOTIMPLEMENTED();
 435   LOG(ERROR) << "OnRenderError()";
 436 }
 437
 438 // Called by AudioPullFifo when more data is necessary.
 439 void WebRtcAudioRenderer::SourceCallback(
 440     int fifo_frame_delay, media::AudioBus* audio_bus) {
 441   base::TimeTicks start_time = base::TimeTicks::Now();
 442   DVLOG(2) << "WebRtcAudioRenderer::SourceCallback("
 443            << fifo_frame_delay << ", "
 444            << audio_bus->frames() << ")";
 445
 446   int output_delay_milliseconds = audio_delay_milliseconds_;
 447   output_delay_milliseconds += fifo_delay_milliseconds_;
 448   DVLOG(2) << "output_delay_milliseconds: " << output_delay_milliseconds;
 449
 450   // We need to keep render data for the |source_| regardless of |state_|,
 451   // otherwise the data will be buffered up inside |source_|.
 452   source_->RenderData(audio_bus, sink_params_.sample_rate(),
 453                       output_delay_milliseconds,
 454                       &current_time_);
 455
 456   // Avoid filling up the audio bus if we are not playing; instead
 457   // return here and ensure that the returned value in Render() is 0.
 458   if (state_ != PLAYING)
 459     audio_bus->Zero();
 460
 461   if (++render_callback_count_ == kNumCallbacksBetweenRenderTimeHistograms) {
 462     base::TimeDelta elapsed = base::TimeTicks::Now() - start_time;
 463     render_callback_count_ = 0;
 464     UMA_HISTOGRAM_TIMES("WebRTC.AudioRenderTimes", elapsed);
 465   }
 466 }
 467
 468 void WebRtcAudioRenderer::UpdateSourceVolume(
 469     webrtc::AudioSourceInterface* source) {
 470   DCHECK(thread_checker_.CalledOnValidThread());
 471
 472   // Note: If there are no playing audio renderers, then the volume will be
 473   // set to 0.0.
 474   float volume = 0.0f;
 475
 476   SourcePlayingStates::iterator entry = source_playing_states_.find(source);
 477   if (entry != source_playing_states_.end()) {
 478     PlayingStates& states = entry->second;
 479     for (PlayingStates::const_iterator it = states.begin();
 480          it != states.end(); ++it) {
 481       if ((*it)->playing())
 482         volume += (*it)->volume();
 483     }
 484   }
 485
 486   // The valid range for volume scaling of a remote webrtc source is
 487   // 0.0-10.0 where 1.0 is no attenuation/boost.
 488   DCHECK(volume >= 0.0f);
 489   if (volume > 10.0f)
 490     volume = 10.0f;
 491
 492   DVLOG(1) << "Setting remote source volume: " << volume;
 493   if (!signaling_thread_->BelongsToCurrentThread()) {
 494     // Libjingle hands out proxy objects in most cases, but the audio source
 495     // object is an exception (bug?).  So, to work around that, we need to make
 496     // sure we call SetVolume on the signaling thread.
 497     signaling_thread_->PostTask(FROM_HERE,
 498         base::Bind(&webrtc::AudioSourceInterface::SetVolume, source, volume));
 499   } else {
 500     source->SetVolume(volume);
 501   }
 502 }
 503
 504 bool WebRtcAudioRenderer::AddPlayingState(
 505     webrtc::AudioSourceInterface* source,
 506     PlayingState* state) {
 507   DCHECK(thread_checker_.CalledOnValidThread());
 508   DCHECK(state->playing());
 509   // Look up or add the |source| to the map.
 510   PlayingStates& array = source_playing_states_[source];
 511   if (std::find(array.begin(), array.end(), state) != array.end())
 512     return false;
 513
 514   array.push_back(state);
 515
 516   return true;
 517 }
 518
 519 bool WebRtcAudioRenderer::RemovePlayingState(
 520     webrtc::AudioSourceInterface* source,
 521     PlayingState* state) {
 522   DCHECK(thread_checker_.CalledOnValidThread());
 523   DCHECK(!state->playing());
 524   SourcePlayingStates::iterator found = source_playing_states_.find(source);
 525   if (found == source_playing_states_.end())
 526     return false;
 527
 528   PlayingStates& array = found->second;
 529   PlayingStates::iterator state_it =
 530       std::find(array.begin(), array.end(), state);
 531   if (state_it == array.end())
 532     return false;
 533
 534   array.erase(state_it);
 535
 536   if (array.empty())
 537     source_playing_states_.erase(found);
 538
 539   return true;
 540 }
 541
 542 void WebRtcAudioRenderer::OnPlayStateChanged(
 543     const scoped_refptr<webrtc::MediaStreamInterface>& media_stream,
 544     PlayingState* state) {
 545   webrtc::AudioTrackVector tracks(media_stream->GetAudioTracks());
 546   for (webrtc::AudioTrackVector::iterator it = tracks.begin();
 547        it != tracks.end(); ++it) {
 548     webrtc::AudioSourceInterface* source = (*it)->GetSource();
 549     DCHECK(source);
 550     if (!state->playing()) {
 551       if (RemovePlayingState(source, state))
 552         EnterPauseState();
 553     } else if (AddPlayingState(source, state)) {
 554       EnterPlayState();
 555     }
 556     UpdateSourceVolume(source);
 557   }
 558 }
 559
 560 }  // namespace content