content/renderer/media/webrtc_audio_device_impl.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_device_impl.h"
   6
   7 #include "base/bind.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/win/windows_version.h"
  11 #include "content/renderer/media/media_stream_audio_processor.h"
  12 #include "content/renderer/media/webrtc_audio_capturer.h"
  13 #include "content/renderer/media/webrtc_audio_renderer.h"
  14 #include "content/renderer/render_thread_impl.h"
  15 #include "media/audio/audio_parameters.h"
  16 #include "media/audio/sample_rates.h"
  17
  18 using media::AudioParameters;
  19 using media::ChannelLayout;
  20
  21 namespace content {
  22
  23 WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()
  24     : ref_count_(0),
  25       audio_transport_callback_(NULL),
  26       output_delay_ms_(0),
  27       initialized_(false),
  28       playing_(false),
  29       recording_(false),
  30       microphone_volume_(0) {
  31   DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()";
  32   // This object can be constructed on either the signaling thread or the main
  33   // thread, so we need to detach these thread checkers here and have them
  34   // initialize automatically when the first methods are called.
  35   signaling_thread_checker_.DetachFromThread();
  36   main_thread_checker_.DetachFromThread();
  37
  38   worker_thread_checker_.DetachFromThread();
  39 }
  40
  41 WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl() {
  42   DVLOG(1) << "WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl()";
  43   DCHECK(main_thread_checker_.CalledOnValidThread());
  44   Terminate();
  45 }
  46
  47 int32_t WebRtcAudioDeviceImpl::AddRef() {
  48   // We can be AddRefed and released on both the UI thread as well as
  49   // libjingle's signaling thread.
  50   return base::subtle::Barrier_AtomicIncrement(&ref_count_, 1);
  51 }
  52
  53 int32_t WebRtcAudioDeviceImpl::Release() {
  54   // We can be AddRefed and released on both the UI thread as well as
  55   // libjingle's signaling thread.
  56   int ret = base::subtle::Barrier_AtomicIncrement(&ref_count_, -1);
  57   if (ret == 0) {
  58     delete this;
  59   }
  60   return ret;
  61 }
  62
  63 void WebRtcAudioDeviceImpl::RenderData(media::AudioBus* audio_bus,
  64                                        int sample_rate,
  65                                        int audio_delay_milliseconds,
  66                                        base::TimeDelta* current_time) {
  67   {
  68     base::AutoLock auto_lock(lock_);
  69     if (!playing_) {
  70       // Force silence to AudioBus after stopping playout in case
  71       // there is lingering audio data in AudioBus.
  72       audio_bus->Zero();
  73       return;
  74     }
  75     DCHECK(audio_transport_callback_);
  76     // Store the reported audio delay locally.
  77     output_delay_ms_ = audio_delay_milliseconds;
  78   }
  79
  80   render_buffer_.resize(audio_bus->frames() * audio_bus->channels());
  81   int frames_per_10_ms = (sample_rate / 100);
  82   int bytes_per_sample = sizeof(render_buffer_[0]);
  83   const int bytes_per_10_ms =
  84       audio_bus->channels() * frames_per_10_ms * bytes_per_sample;
  85   DCHECK_EQ(audio_bus->frames() % frames_per_10_ms, 0);
  86
  87   // Get audio frames in blocks of 10 milliseconds from the registered
  88   // webrtc::AudioTransport source. Keep reading until our internal buffer
  89   // is full.
  90   int accumulated_audio_frames = 0;
  91   int16* audio_data = &render_buffer_[0];
  92   while (accumulated_audio_frames < audio_bus->frames()) {
  93     // Get 10ms and append output to temporary byte buffer.
  94     int64_t elapsed_time_ms = -1;
  95     int64_t ntp_time_ms = -1;
  96     static const int kBitsPerByte = 8;
  97     audio_transport_callback_->PullRenderData(bytes_per_sample * kBitsPerByte,
  98                                               sample_rate,
  99                                               audio_bus->channels(),
 100                                               frames_per_10_ms,
 101                                               audio_data,
 102                                               &elapsed_time_ms,
 103                                               &ntp_time_ms);
 104     accumulated_audio_frames += frames_per_10_ms;
 105     if (elapsed_time_ms >= 0) {
 106       *current_time = base::TimeDelta::FromMilliseconds(elapsed_time_ms);
 107     }
 108     audio_data += bytes_per_10_ms;
 109   }
 110
 111   // De-interleave each channel and convert to 32-bit floating-point
 112   // with nominal range -1.0 -> +1.0 to match the callback format.
 113   audio_bus->FromInterleaved(&render_buffer_[0],
 114                              audio_bus->frames(),
 115                              bytes_per_sample);
 116
 117   // Pass the render data to the playout sinks.
 118   base::AutoLock auto_lock(lock_);
 119   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 120        it != playout_sinks_.end(); ++it) {
 121     (*it)->OnPlayoutData(audio_bus, sample_rate, audio_delay_milliseconds);
 122   }
 123 }
 124
 125 void WebRtcAudioDeviceImpl::RemoveAudioRenderer(WebRtcAudioRenderer* renderer) {
 126   DCHECK(main_thread_checker_.CalledOnValidThread());
 127   base::AutoLock auto_lock(lock_);
 128   DCHECK_EQ(renderer, renderer_.get());
 129   // Notify the playout sink of the change.
 130   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 131        it != playout_sinks_.end(); ++it) {
 132     (*it)->OnPlayoutDataSourceChanged();
 133   }
 134
 135   renderer_ = NULL;
 136 }
 137
 138 int32_t WebRtcAudioDeviceImpl::RegisterAudioCallback(
 139     webrtc::AudioTransport* audio_callback) {
 140   DVLOG(1) << "WebRtcAudioDeviceImpl::RegisterAudioCallback()";
 141   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 142   base::AutoLock lock(lock_);
 143   DCHECK_EQ(audio_transport_callback_ == NULL, audio_callback != NULL);
 144   audio_transport_callback_ = audio_callback;
 145   return 0;
 146 }
 147
 148 int32_t WebRtcAudioDeviceImpl::Init() {
 149   DVLOG(1) << "WebRtcAudioDeviceImpl::Init()";
 150   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 151
 152   // We need to return a success to continue the initialization of WebRtc VoE
 153   // because failure on the capturer_ initialization should not prevent WebRTC
 154   // from working. See issue http://crbug.com/144421 for details.
 155   initialized_ = true;
 156
 157   return 0;
 158 }
 159
 160 int32_t WebRtcAudioDeviceImpl::Terminate() {
 161   DVLOG(1) << "WebRtcAudioDeviceImpl::Terminate()";
 162   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 163
 164   // Calling Terminate() multiple times in a row is OK.
 165   if (!initialized_)
 166     return 0;
 167
 168   StopRecording();
 169   StopPlayout();
 170
 171   DCHECK(!renderer_.get() || !renderer_->IsStarted())
 172       << "The shared audio renderer shouldn't be running";
 173
 174   // Stop all the capturers to ensure no further OnData() and
 175   // RemoveAudioCapturer() callback.
 176   // Cache the capturers in a local list since WebRtcAudioCapturer::Stop()
 177   // will trigger RemoveAudioCapturer() callback.
 178   CapturerList capturers;
 179   capturers.swap(capturers_);
 180   for (CapturerList::const_iterator iter = capturers.begin();
 181        iter != capturers.end(); ++iter) {
 182     (*iter)->Stop();
 183   }
 184
 185   initialized_ = false;
 186   return 0;
 187 }
 188
 189 bool WebRtcAudioDeviceImpl::Initialized() const {
 190   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 191   return initialized_;
 192 }
 193
 194 int32_t WebRtcAudioDeviceImpl::PlayoutIsAvailable(bool* available) {
 195   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 196   *available = initialized_;
 197   return 0;
 198 }
 199
 200 bool WebRtcAudioDeviceImpl::PlayoutIsInitialized() const {
 201   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 202   return initialized_;
 203 }
 204
 205 int32_t WebRtcAudioDeviceImpl::RecordingIsAvailable(bool* available) {
 206   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 207   base::AutoLock auto_lock(lock_);
 208   *available = (!capturers_.empty());
 209   return 0;
 210 }
 211
 212 bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const {
 213   DVLOG(1) << "WebRtcAudioDeviceImpl::RecordingIsInitialized()";
 214   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 215   base::AutoLock auto_lock(lock_);
 216   return (!capturers_.empty());
 217 }
 218
 219 int32_t WebRtcAudioDeviceImpl::StartPlayout() {
 220   DVLOG(1) << "WebRtcAudioDeviceImpl::StartPlayout()";
 221   DCHECK(worker_thread_checker_.CalledOnValidThread());
 222   base::AutoLock auto_lock(lock_);
 223   if (!audio_transport_callback_) {
 224     LOG(ERROR) << "Audio transport is missing";
 225     return 0;
 226   }
 227
 228   // webrtc::VoiceEngine assumes that it is OK to call Start() twice and
 229   // that the call is ignored the second time.
 230   playing_ = true;
 231   return 0;
 232 }
 233
 234 int32_t WebRtcAudioDeviceImpl::StopPlayout() {
 235   DVLOG(1) << "WebRtcAudioDeviceImpl::StopPlayout()";
 236   DCHECK(initialized_);
 237   // Can be called both from the worker thread (e.g. when called from webrtc)
 238   // or the signaling thread (e.g. when we call it ourselves internally).
 239   // The order in this check is important so that we won't incorrectly
 240   // initialize worker_thread_checker_ on the signaling thread.
 241   DCHECK(signaling_thread_checker_.CalledOnValidThread() ||
 242          worker_thread_checker_.CalledOnValidThread());
 243   base::AutoLock auto_lock(lock_);
 244   // webrtc::VoiceEngine assumes that it is OK to call Stop() multiple times.
 245   playing_ = false;
 246   return 0;
 247 }
 248
 249 bool WebRtcAudioDeviceImpl::Playing() const {
 250   DCHECK(worker_thread_checker_.CalledOnValidThread());
 251   base::AutoLock auto_lock(lock_);
 252   return playing_;
 253 }
 254
 255 int32_t WebRtcAudioDeviceImpl::StartRecording() {
 256   DVLOG(1) << "WebRtcAudioDeviceImpl::StartRecording()";
 257   DCHECK(worker_thread_checker_.CalledOnValidThread());
 258   DCHECK(initialized_);
 259   base::AutoLock auto_lock(lock_);
 260   if (!audio_transport_callback_) {
 261     LOG(ERROR) << "Audio transport is missing";
 262     return -1;
 263   }
 264
 265   recording_ = true;
 266
 267   return 0;
 268 }
 269
 270 int32_t WebRtcAudioDeviceImpl::StopRecording() {
 271   DVLOG(1) << "WebRtcAudioDeviceImpl::StopRecording()";
 272   DCHECK(initialized_);
 273   // Can be called both from the worker thread (e.g. when called from webrtc)
 274   // or the signaling thread (e.g. when we call it ourselves internally).
 275   // The order in this check is important so that we won't incorrectly
 276   // initialize worker_thread_checker_ on the signaling thread.
 277   DCHECK(signaling_thread_checker_.CalledOnValidThread() ||
 278          worker_thread_checker_.CalledOnValidThread());
 279
 280   base::AutoLock auto_lock(lock_);
 281   recording_ = false;
 282   return 0;
 283 }
 284
 285 bool WebRtcAudioDeviceImpl::Recording() const {
 286   DCHECK(worker_thread_checker_.CalledOnValidThread());
 287   base::AutoLock auto_lock(lock_);
 288   return recording_;
 289 }
 290
 291 int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) {
 292   DVLOG(1) << "WebRtcAudioDeviceImpl::SetMicrophoneVolume(" << volume << ")";
 293   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 294   DCHECK(initialized_);
 295
 296   // Only one microphone is supported at the moment, which is represented by
 297   // the default capturer.
 298   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 299   if (!capturer.get())
 300     return -1;
 301
 302   capturer->SetVolume(volume);
 303   return 0;
 304 }
 305
 306 // TODO(henrika): sort out calling thread once we start using this API.
 307 int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const {
 308   DVLOG(1) << "WebRtcAudioDeviceImpl::MicrophoneVolume()";
 309   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 310   // We only support one microphone now, which is accessed via the default
 311   // capturer.
 312   DCHECK(initialized_);
 313   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 314   if (!capturer.get())
 315     return -1;
 316
 317   *volume = static_cast<uint32_t>(capturer->Volume());
 318
 319   return 0;
 320 }
 321
 322 int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const {
 323   DCHECK(initialized_);
 324   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 325   *max_volume = kMaxVolumeLevel;
 326   return 0;
 327 }
 328
 329 int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const {
 330   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 331   *min_volume = 0;
 332   return 0;
 333 }
 334
 335 int32_t WebRtcAudioDeviceImpl::StereoPlayoutIsAvailable(bool* available) const {
 336   DCHECK(initialized_);
 337   // This method is called during initialization on the signaling thread and
 338   // then later on the worker thread.  Due to this we cannot DCHECK on what
 339   // thread we're on since it might incorrectly initialize the
 340   // worker_thread_checker_.
 341   base::AutoLock auto_lock(lock_);
 342   *available = renderer_.get() && renderer_->channels() == 2;
 343   return 0;
 344 }
 345
 346 int32_t WebRtcAudioDeviceImpl::StereoRecordingIsAvailable(
 347     bool* available) const {
 348   DCHECK(initialized_);
 349   // This method is called during initialization on the signaling thread and
 350   // then later on the worker thread.  Due to this we cannot DCHECK on what
 351   // thread we're on since it might incorrectly initialize the
 352   // worker_thread_checker_.
 353
 354   // TODO(xians): These kind of hardware methods do not make much sense since we
 355   // support multiple sources. Remove or figure out new APIs for such methods.
 356   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 357   if (!capturer.get())
 358     return -1;
 359
 360   *available = (capturer->source_audio_parameters().channels() == 2);
 361   return 0;
 362 }
 363
 364 int32_t WebRtcAudioDeviceImpl::PlayoutDelay(uint16_t* delay_ms) const {
 365   DCHECK(worker_thread_checker_.CalledOnValidThread());
 366   base::AutoLock auto_lock(lock_);
 367   *delay_ms = static_cast<uint16_t>(output_delay_ms_);
 368   return 0;
 369 }
 370
 371 int32_t WebRtcAudioDeviceImpl::RecordingDelay(uint16_t* delay_ms) const {
 372   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 373
 374   // There is no way to report a correct delay value to WebRTC since there
 375   // might be multiple WebRtcAudioCapturer instances.
 376   NOTREACHED();
 377   return -1;
 378 }
 379
 380 int32_t WebRtcAudioDeviceImpl::RecordingSampleRate(
 381     uint32_t* sample_rate) const {
 382   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 383   // We use the default capturer as the recording sample rate.
 384   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 385   if (!capturer.get())
 386     return -1;
 387
 388   *sample_rate = static_cast<uint32_t>(
 389       capturer->source_audio_parameters().sample_rate());
 390   return 0;
 391 }
 392
 393 int32_t WebRtcAudioDeviceImpl::PlayoutSampleRate(
 394     uint32_t* sample_rate) const {
 395   DCHECK(signaling_thread_checker_.CalledOnValidThread());
 396   *sample_rate = renderer_.get() ? renderer_->sample_rate() : 0;
 397   return 0;
 398 }
 399
 400 bool WebRtcAudioDeviceImpl::SetAudioRenderer(WebRtcAudioRenderer* renderer) {
 401   DCHECK(main_thread_checker_.CalledOnValidThread());
 402   DCHECK(renderer);
 403
 404   // Here we acquire |lock_| in order to protect the internal state.
 405   {
 406     base::AutoLock auto_lock(lock_);
 407     if (renderer_.get())
 408       return false;
 409   }
 410
 411   // We release |lock_| here because invoking |renderer|->Initialize while
 412   // holding |lock_| would result in locks taken in the sequence
 413   // (|this->lock_|,  |renderer->lock_|) while another thread (i.e, the
 414   // AudioOutputDevice thread) might concurrently invoke a renderer method,
 415   // which can itself invoke a method from |this|, resulting in locks taken in
 416   // the sequence (|renderer->lock_|, |this->lock_|) in that thread.
 417   // This order discrepancy can cause a deadlock (see Issue 433993).
 418   // However, we do not need to hold |this->lock_| in order to invoke
 419   // |renderer|->Initialize, since it does not involve any unprotected access to
 420   // the internal state of |this|.
 421   if (!renderer->Initialize(this))
 422     return false;
 423
 424   // We acquire |lock_| again and assert our precondition, since we are
 425   // accessing the internal state again.
 426   base::AutoLock auto_lock(lock_);
 427   DCHECK(!renderer_.get());
 428   renderer_ = renderer;
 429   return true;
 430 }
 431
 432 void WebRtcAudioDeviceImpl::AddAudioCapturer(
 433     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 434   DCHECK(main_thread_checker_.CalledOnValidThread());
 435   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 436   DCHECK(capturer.get());
 437   DCHECK(!capturer->device_id().empty());
 438
 439   base::AutoLock auto_lock(lock_);
 440   DCHECK(std::find(capturers_.begin(), capturers_.end(), capturer) ==
 441       capturers_.end());
 442   capturers_.push_back(capturer);
 443 }
 444
 445 void WebRtcAudioDeviceImpl::RemoveAudioCapturer(
 446     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 447   DCHECK(main_thread_checker_.CalledOnValidThread());
 448   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 449   DCHECK(capturer.get());
 450   base::AutoLock auto_lock(lock_);
 451   capturers_.remove(capturer);
 452 }
 453
 454 scoped_refptr<WebRtcAudioCapturer>
 455 WebRtcAudioDeviceImpl::GetDefaultCapturer() const {
 456   // Called on the signaling thread (during initialization), worker
 457   // thread during capture or main thread for a WebAudio source.
 458   // We can't DCHECK on those three checks here since GetDefaultCapturer
 459   // may be the first call and therefore could incorrectly initialize the
 460   // thread checkers.
 461   DCHECK(initialized_);
 462   base::AutoLock auto_lock(lock_);
 463   // Use the last |capturer| which is from the latest getUserMedia call as
 464   // the default capture device.
 465   return capturers_.empty() ? NULL : capturers_.back();
 466 }
 467
 468 void WebRtcAudioDeviceImpl::AddPlayoutSink(
 469     WebRtcPlayoutDataSource::Sink* sink) {
 470   DCHECK(main_thread_checker_.CalledOnValidThread());
 471   DCHECK(sink);
 472   base::AutoLock auto_lock(lock_);
 473   DCHECK(std::find(playout_sinks_.begin(), playout_sinks_.end(), sink) ==
 474       playout_sinks_.end());
 475   playout_sinks_.push_back(sink);
 476 }
 477
 478 void WebRtcAudioDeviceImpl::RemovePlayoutSink(
 479     WebRtcPlayoutDataSource::Sink* sink) {
 480   DCHECK(main_thread_checker_.CalledOnValidThread());
 481   DCHECK(sink);
 482   base::AutoLock auto_lock(lock_);
 483   playout_sinks_.remove(sink);
 484 }
 485
 486 bool WebRtcAudioDeviceImpl::GetAuthorizedDeviceInfoForAudioRenderer(
 487     int* session_id,
 488     int* output_sample_rate,
 489     int* output_frames_per_buffer) {
 490   DCHECK(main_thread_checker_.CalledOnValidThread());
 491   base::AutoLock lock(lock_);
 492   // If there is no capturer or there are more than one open capture devices,
 493   // return false.
 494   if (capturers_.size() != 1)
 495     return false;
 496
 497   return capturers_.back()->GetPairedOutputParameters(
 498       session_id, output_sample_rate, output_frames_per_buffer);
 499 }
 500
 501 }  // namespace content