content/renderer/media/webrtc_audio_device_impl.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/webrtc_audio_device_impl.h"
   6
   7 #include "base/bind.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/strings/string_util.h"
  10 #include "base/win/windows_version.h"
  11 #include "content/renderer/media/media_stream_audio_processor.h"
  12 #include "content/renderer/media/webrtc_audio_capturer.h"
  13 #include "content/renderer/media/webrtc_audio_renderer.h"
  14 #include "content/renderer/render_thread_impl.h"
  15 #include "media/audio/audio_parameters.h"
  16 #include "media/audio/sample_rates.h"
  17
  18 using media::AudioParameters;
  19 using media::ChannelLayout;
  20
  21 namespace content {
  22
  23 WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()
  24     : ref_count_(0),
  25       audio_transport_callback_(NULL),
  26       input_delay_ms_(0),
  27       output_delay_ms_(0),
  28       initialized_(false),
  29       playing_(false),
  30       recording_(false),
  31       microphone_volume_(0),
  32       is_audio_track_processing_enabled_(
  33           MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled()) {
  34   DVLOG(1) << "WebRtcAudioDeviceImpl::WebRtcAudioDeviceImpl()";
  35 }
  36
  37 WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl() {
  38   DVLOG(1) << "WebRtcAudioDeviceImpl::~WebRtcAudioDeviceImpl()";
  39   DCHECK(thread_checker_.CalledOnValidThread());
  40   Terminate();
  41 }
  42
  43 int32_t WebRtcAudioDeviceImpl::AddRef() {
  44   DCHECK(thread_checker_.CalledOnValidThread());
  45   return base::subtle::Barrier_AtomicIncrement(&ref_count_, 1);
  46 }
  47
  48 int32_t WebRtcAudioDeviceImpl::Release() {
  49   DCHECK(thread_checker_.CalledOnValidThread());
  50   int ret = base::subtle::Barrier_AtomicIncrement(&ref_count_, -1);
  51   if (ret == 0) {
  52     delete this;
  53   }
  54   return ret;
  55 }
  56 int WebRtcAudioDeviceImpl::OnData(const int16* audio_data,
  57                                   int sample_rate,
  58                                   int number_of_channels,
  59                                   int number_of_frames,
  60                                   const std::vector<int>& channels,
  61                                   int audio_delay_milliseconds,
  62                                   int current_volume,
  63                                   bool need_audio_processing,
  64                                   bool key_pressed) {
  65   int total_delay_ms = 0;
  66   {
  67     base::AutoLock auto_lock(lock_);
  68     // Return immediately when not recording or |channels| is empty.
  69     // See crbug.com/274017: renderer crash dereferencing invalid channels[0].
  70     if (!recording_ || channels.empty())
  71       return 0;
  72
  73     // Store the reported audio delay locally.
  74     input_delay_ms_ = audio_delay_milliseconds;
  75     total_delay_ms = input_delay_ms_ + output_delay_ms_;
  76     DVLOG(2) << "total delay: " << input_delay_ms_ + output_delay_ms_;
  77   }
  78
  79   // Write audio frames in blocks of 10 milliseconds to the registered
  80   // webrtc::AudioTransport sink. Keep writing until our internal byte
  81   // buffer is empty.
  82   const int16* audio_buffer = audio_data;
  83   const int frames_per_10_ms = (sample_rate / 100);
  84   CHECK_EQ(number_of_frames % frames_per_10_ms, 0);
  85   int accumulated_audio_frames = 0;
  86   uint32_t new_volume = 0;
  87
  88   // The lock here is to protect a race in the resampler inside webrtc when
  89   // there are more than one input stream calling OnData(), which can happen
  90   // when the users setup two getUserMedia, one for the microphone, another
  91   // for WebAudio. Currently we don't have a better way to fix it except for
  92   // adding a lock here to sequence the call.
  93   // TODO(xians): Remove this workaround after we move the
  94   // webrtc::AudioProcessing module to Chrome. See http://crbug/264611 for
  95   // details.
  96   base::AutoLock auto_lock(capture_callback_lock_);
  97   while (accumulated_audio_frames < number_of_frames) {
  98     // Deliver 10ms of recorded 16-bit linear PCM audio.
  99     int new_mic_level = audio_transport_callback_->OnDataAvailable(
 100         &channels[0],
 101         channels.size(),
 102         audio_buffer,
 103         sample_rate,
 104         number_of_channels,
 105         frames_per_10_ms,
 106         total_delay_ms,
 107         current_volume,
 108         key_pressed,
 109         need_audio_processing);
 110
 111     accumulated_audio_frames += frames_per_10_ms;
 112     audio_buffer += frames_per_10_ms * number_of_channels;
 113
 114     // The latest non-zero new microphone level will be returned.
 115     if (new_mic_level)
 116       new_volume = new_mic_level;
 117   }
 118
 119   return new_volume;
 120 }
 121
 122 void WebRtcAudioDeviceImpl::OnSetFormat(
 123     const media::AudioParameters& params) {
 124   DVLOG(1) << "WebRtcAudioDeviceImpl::OnSetFormat()";
 125 }
 126
 127 void WebRtcAudioDeviceImpl::RenderData(media::AudioBus* audio_bus,
 128                                        int sample_rate,
 129                                        int audio_delay_milliseconds,
 130                                        base::TimeDelta* current_time) {
 131   render_buffer_.resize(audio_bus->frames() * audio_bus->channels());
 132
 133   {
 134     base::AutoLock auto_lock(lock_);
 135     DCHECK(audio_transport_callback_);
 136     // Store the reported audio delay locally.
 137     output_delay_ms_ = audio_delay_milliseconds;
 138   }
 139
 140   int frames_per_10_ms = (sample_rate / 100);
 141   int bytes_per_sample = sizeof(render_buffer_[0]);
 142   const int bytes_per_10_ms =
 143       audio_bus->channels() * frames_per_10_ms * bytes_per_sample;
 144   DCHECK_EQ(audio_bus->frames() % frames_per_10_ms, 0);
 145
 146   // Get audio frames in blocks of 10 milliseconds from the registered
 147   // webrtc::AudioTransport source. Keep reading until our internal buffer
 148   // is full.
 149   uint32_t num_audio_frames = 0;
 150   int accumulated_audio_frames = 0;
 151   int16* audio_data = &render_buffer_[0];
 152   while (accumulated_audio_frames < audio_bus->frames()) {
 153     // Get 10ms and append output to temporary byte buffer.
 154     int64_t elapsed_time_ms = -1;
 155     int64_t ntp_time_ms = -1;
 156     if (is_audio_track_processing_enabled_) {
 157       // When audio processing is enabled in the audio track, we use
 158       // PullRenderData() instead of NeedMorePlayData() to avoid passing the
 159       // render data to the APM in WebRTC as reference signal for echo
 160       // cancellation.
 161       static const int kBitsPerByte = 8;
 162       audio_transport_callback_->PullRenderData(bytes_per_sample * kBitsPerByte,
 163                                                 sample_rate,
 164                                                 audio_bus->channels(),
 165                                                 frames_per_10_ms,
 166                                                 audio_data,
 167                                                 &elapsed_time_ms,
 168                                                 &ntp_time_ms);
 169       accumulated_audio_frames += frames_per_10_ms;
 170     } else {
 171       // TODO(xians): Remove the following code after the APM in WebRTC is
 172       // deprecated.
 173       audio_transport_callback_->NeedMorePlayData(frames_per_10_ms,
 174                                                   bytes_per_sample,
 175                                                   audio_bus->channels(),
 176                                                   sample_rate,
 177                                                   audio_data,
 178                                                   num_audio_frames,
 179                                                   &elapsed_time_ms,
 180                                                   &ntp_time_ms);
 181       accumulated_audio_frames += num_audio_frames;
 182     }
 183     if (elapsed_time_ms >= 0) {
 184       *current_time = base::TimeDelta::FromMilliseconds(elapsed_time_ms);
 185     }
 186     audio_data += bytes_per_10_ms;
 187   }
 188
 189   // De-interleave each channel and convert to 32-bit floating-point
 190   // with nominal range -1.0 -> +1.0 to match the callback format.
 191   audio_bus->FromInterleaved(&render_buffer_[0],
 192                              audio_bus->frames(),
 193                              bytes_per_sample);
 194
 195   // Pass the render data to the playout sinks.
 196   base::AutoLock auto_lock(lock_);
 197   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 198        it != playout_sinks_.end(); ++it) {
 199     (*it)->OnPlayoutData(audio_bus, sample_rate, audio_delay_milliseconds);
 200   }
 201 }
 202
 203 void WebRtcAudioDeviceImpl::RemoveAudioRenderer(WebRtcAudioRenderer* renderer) {
 204   DCHECK(thread_checker_.CalledOnValidThread());
 205   DCHECK_EQ(renderer, renderer_.get());
 206   base::AutoLock auto_lock(lock_);
 207   // Notify the playout sink of the change.
 208   for (PlayoutDataSinkList::const_iterator it = playout_sinks_.begin();
 209        it != playout_sinks_.end(); ++it) {
 210     (*it)->OnPlayoutDataSourceChanged();
 211   }
 212
 213   renderer_ = NULL;
 214   playing_ = false;
 215 }
 216
 217 int32_t WebRtcAudioDeviceImpl::RegisterAudioCallback(
 218     webrtc::AudioTransport* audio_callback) {
 219   DVLOG(1) << "WebRtcAudioDeviceImpl::RegisterAudioCallback()";
 220   DCHECK(thread_checker_.CalledOnValidThread());
 221   DCHECK_EQ(audio_transport_callback_ == NULL, audio_callback != NULL);
 222   audio_transport_callback_ = audio_callback;
 223   return 0;
 224 }
 225
 226 int32_t WebRtcAudioDeviceImpl::Init() {
 227   DVLOG(1) << "WebRtcAudioDeviceImpl::Init()";
 228   DCHECK(thread_checker_.CalledOnValidThread());
 229
 230   // We need to return a success to continue the initialization of WebRtc VoE
 231   // because failure on the capturer_ initialization should not prevent WebRTC
 232   // from working. See issue http://crbug.com/144421 for details.
 233   initialized_ = true;
 234
 235   return 0;
 236 }
 237
 238 int32_t WebRtcAudioDeviceImpl::Terminate() {
 239   DVLOG(1) << "WebRtcAudioDeviceImpl::Terminate()";
 240   DCHECK(thread_checker_.CalledOnValidThread());
 241
 242   // Calling Terminate() multiple times in a row is OK.
 243   if (!initialized_)
 244     return 0;
 245
 246   StopRecording();
 247   StopPlayout();
 248
 249   DCHECK(!renderer_.get() || !renderer_->IsStarted())
 250       << "The shared audio renderer shouldn't be running";
 251
 252   // Stop all the capturers to ensure no further OnData() and
 253   // RemoveAudioCapturer() callback.
 254   // Cache the capturers in a local list since WebRtcAudioCapturer::Stop()
 255   // will trigger RemoveAudioCapturer() callback.
 256   CapturerList capturers;
 257   capturers.swap(capturers_);
 258   for (CapturerList::const_iterator iter = capturers.begin();
 259        iter != capturers.end(); ++iter) {
 260     (*iter)->Stop();
 261   }
 262
 263   initialized_ = false;
 264   return 0;
 265 }
 266
 267 bool WebRtcAudioDeviceImpl::Initialized() const {
 268   return initialized_;
 269 }
 270
 271 int32_t WebRtcAudioDeviceImpl::PlayoutIsAvailable(bool* available) {
 272   *available = initialized_;
 273   return 0;
 274 }
 275
 276 bool WebRtcAudioDeviceImpl::PlayoutIsInitialized() const {
 277   return initialized_;
 278 }
 279
 280 int32_t WebRtcAudioDeviceImpl::RecordingIsAvailable(bool* available) {
 281   *available = (!capturers_.empty());
 282   return 0;
 283 }
 284
 285 bool WebRtcAudioDeviceImpl::RecordingIsInitialized() const {
 286   DVLOG(1) << "WebRtcAudioDeviceImpl::RecordingIsInitialized()";
 287   DCHECK(thread_checker_.CalledOnValidThread());
 288   return (!capturers_.empty());
 289 }
 290
 291 int32_t WebRtcAudioDeviceImpl::StartPlayout() {
 292   DVLOG(1) << "WebRtcAudioDeviceImpl::StartPlayout()";
 293   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
 294   {
 295     base::AutoLock auto_lock(lock_);
 296     if (!audio_transport_callback_)
 297       return 0;
 298   }
 299
 300   if (playing_) {
 301     // webrtc::VoiceEngine assumes that it is OK to call Start() twice and
 302     // that the call is ignored the second time.
 303     return 0;
 304   }
 305
 306   playing_ = true;
 307   return 0;
 308 }
 309
 310 int32_t WebRtcAudioDeviceImpl::StopPlayout() {
 311   DVLOG(1) << "WebRtcAudioDeviceImpl::StopPlayout()";
 312   if (!playing_) {
 313     // webrtc::VoiceEngine assumes that it is OK to call Stop() just in case.
 314     return 0;
 315   }
 316
 317   playing_ = false;
 318   return 0;
 319 }
 320
 321 bool WebRtcAudioDeviceImpl::Playing() const {
 322   return playing_;
 323 }
 324
 325 int32_t WebRtcAudioDeviceImpl::StartRecording() {
 326   DVLOG(1) << "WebRtcAudioDeviceImpl::StartRecording()";
 327   DCHECK(initialized_);
 328   LOG_IF(ERROR, !audio_transport_callback_) << "Audio transport is missing";
 329   if (!audio_transport_callback_) {
 330     return -1;
 331   }
 332
 333   {
 334     base::AutoLock auto_lock(lock_);
 335     if (recording_)
 336       return 0;
 337
 338     recording_ = true;
 339   }
 340
 341   return 0;
 342 }
 343
 344 int32_t WebRtcAudioDeviceImpl::StopRecording() {
 345   DVLOG(1) << "WebRtcAudioDeviceImpl::StopRecording()";
 346   {
 347     base::AutoLock auto_lock(lock_);
 348     if (!recording_)
 349       return 0;
 350
 351     recording_ = false;
 352   }
 353
 354   return 0;
 355 }
 356
 357 bool WebRtcAudioDeviceImpl::Recording() const {
 358   base::AutoLock auto_lock(lock_);
 359   return recording_;
 360 }
 361
 362 int32_t WebRtcAudioDeviceImpl::SetMicrophoneVolume(uint32_t volume) {
 363   DVLOG(1) << "WebRtcAudioDeviceImpl::SetMicrophoneVolume(" << volume << ")";
 364   DCHECK(initialized_);
 365
 366   // Only one microphone is supported at the moment, which is represented by
 367   // the default capturer.
 368   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 369   if (!capturer.get())
 370     return -1;
 371
 372   capturer->SetVolume(volume);
 373   return 0;
 374 }
 375
 376 // TODO(henrika): sort out calling thread once we start using this API.
 377 int32_t WebRtcAudioDeviceImpl::MicrophoneVolume(uint32_t* volume) const {
 378   DVLOG(1) << "WebRtcAudioDeviceImpl::MicrophoneVolume()";
 379   // We only support one microphone now, which is accessed via the default
 380   // capturer.
 381   DCHECK(initialized_);
 382   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 383   if (!capturer.get())
 384     return -1;
 385
 386   *volume = static_cast<uint32_t>(capturer->Volume());
 387
 388   return 0;
 389 }
 390
 391 int32_t WebRtcAudioDeviceImpl::MaxMicrophoneVolume(uint32_t* max_volume) const {
 392   DCHECK(initialized_);
 393   *max_volume = kMaxVolumeLevel;
 394   return 0;
 395 }
 396
 397 int32_t WebRtcAudioDeviceImpl::MinMicrophoneVolume(uint32_t* min_volume) const {
 398   *min_volume = 0;
 399   return 0;
 400 }
 401
 402 int32_t WebRtcAudioDeviceImpl::StereoPlayoutIsAvailable(bool* available) const {
 403   DCHECK(initialized_);
 404   *available = renderer_.get() && renderer_->channels() == 2;
 405   return 0;
 406 }
 407
 408 int32_t WebRtcAudioDeviceImpl::StereoRecordingIsAvailable(
 409     bool* available) const {
 410   DCHECK(initialized_);
 411   // TODO(xians): These kind of hardware methods do not make much sense since we
 412   // support multiple sources. Remove or figure out new APIs for such methods.
 413   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 414   if (!capturer.get())
 415     return -1;
 416
 417   *available = (capturer->source_audio_parameters().channels() == 2);
 418   return 0;
 419 }
 420
 421 int32_t WebRtcAudioDeviceImpl::PlayoutDelay(uint16_t* delay_ms) const {
 422   base::AutoLock auto_lock(lock_);
 423   *delay_ms = static_cast<uint16_t>(output_delay_ms_);
 424   return 0;
 425 }
 426
 427 int32_t WebRtcAudioDeviceImpl::RecordingDelay(uint16_t* delay_ms) const {
 428   base::AutoLock auto_lock(lock_);
 429   *delay_ms = static_cast<uint16_t>(input_delay_ms_);
 430   return 0;
 431 }
 432
 433 int32_t WebRtcAudioDeviceImpl::RecordingSampleRate(
 434     uint32_t* sample_rate) const {
 435   // We use the default capturer as the recording sample rate.
 436   scoped_refptr<WebRtcAudioCapturer> capturer(GetDefaultCapturer());
 437   if (!capturer.get())
 438     return -1;
 439
 440   *sample_rate = static_cast<uint32_t>(
 441       capturer->source_audio_parameters().sample_rate());
 442   return 0;
 443 }
 444
 445 int32_t WebRtcAudioDeviceImpl::PlayoutSampleRate(
 446     uint32_t* sample_rate) const {
 447   *sample_rate = renderer_.get() ? renderer_->sample_rate() : 0;
 448   return 0;
 449 }
 450
 451 bool WebRtcAudioDeviceImpl::SetAudioRenderer(WebRtcAudioRenderer* renderer) {
 452   DCHECK(thread_checker_.CalledOnValidThread());
 453   DCHECK(renderer);
 454
 455   base::AutoLock auto_lock(lock_);
 456   if (renderer_.get())
 457     return false;
 458
 459   if (!renderer->Initialize(this))
 460     return false;
 461
 462   renderer_ = renderer;
 463   return true;
 464 }
 465
 466 void WebRtcAudioDeviceImpl::AddAudioCapturer(
 467     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 468   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 469   DCHECK(thread_checker_.CalledOnValidThread());
 470   DCHECK(capturer.get());
 471   DCHECK(!capturer->device_id().empty());
 472   {
 473     base::AutoLock auto_lock(lock_);
 474     DCHECK(std::find(capturers_.begin(), capturers_.end(), capturer) ==
 475         capturers_.end());
 476     capturers_.push_back(capturer);
 477   }
 478 }
 479
 480 void WebRtcAudioDeviceImpl::RemoveAudioCapturer(
 481     const scoped_refptr<WebRtcAudioCapturer>& capturer) {
 482   DVLOG(1) << "WebRtcAudioDeviceImpl::AddAudioCapturer()";
 483   DCHECK(thread_checker_.CalledOnValidThread());
 484   DCHECK(capturer.get());
 485   base::AutoLock auto_lock(lock_);
 486   capturers_.remove(capturer);
 487 }
 488
 489 scoped_refptr<WebRtcAudioCapturer>
 490 WebRtcAudioDeviceImpl::GetDefaultCapturer() const {
 491   base::AutoLock auto_lock(lock_);
 492   // Use the last |capturer| which is from the latest getUserMedia call as
 493   // the default capture device.
 494   return capturers_.empty() ? NULL : capturers_.back();
 495 }
 496
 497 void WebRtcAudioDeviceImpl::AddPlayoutSink(
 498     WebRtcPlayoutDataSource::Sink* sink) {
 499   DCHECK(thread_checker_.CalledOnValidThread());
 500   DCHECK(sink);
 501   base::AutoLock auto_lock(lock_);
 502   DCHECK(std::find(playout_sinks_.begin(), playout_sinks_.end(), sink) ==
 503       playout_sinks_.end());
 504   playout_sinks_.push_back(sink);
 505 }
 506
 507 void WebRtcAudioDeviceImpl::RemovePlayoutSink(
 508     WebRtcPlayoutDataSource::Sink* sink) {
 509   DCHECK(thread_checker_.CalledOnValidThread());
 510   DCHECK(sink);
 511   base::AutoLock auto_lock(lock_);
 512   playout_sinks_.remove(sink);
 513 }
 514
 515 bool WebRtcAudioDeviceImpl::GetAuthorizedDeviceInfoForAudioRenderer(
 516     int* session_id,
 517     int* output_sample_rate,
 518     int* output_frames_per_buffer) {
 519   DCHECK(thread_checker_.CalledOnValidThread());
 520   // If there is no capturer or there are more than one open capture devices,
 521   // return false.
 522   if (capturers_.empty() || capturers_.size() > 1)
 523     return false;
 524
 525   return GetDefaultCapturer()->GetPairedOutputParameters(
 526       session_id, output_sample_rate, output_frames_per_buffer);
 527 }
 528
 529 }  // namespace content