content/renderer/media/media_stream_audio_processor.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/media_stream_audio_processor.h"
   6
   7 #include "base/command_line.h"
   8 #include "base/metrics/field_trial.h"
   9 #include "base/metrics/histogram.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "content/public/common/content_switches.h"
  12 #include "content/renderer/media/media_stream_audio_processor_options.h"
  13 #include "content/renderer/media/rtc_media_constraints.h"
  14 #include "content/renderer/media/webrtc_audio_device_impl.h"
  15 #include "media/audio/audio_parameters.h"
  16 #include "media/base/audio_converter.h"
  17 #include "media/base/audio_fifo.h"
  18 #include "media/base/channel_layout.h"
  19 #include "third_party/WebKit/public/platform/WebMediaConstraints.h"
  20 #include "third_party/libjingle/source/talk/app/webrtc/mediaconstraintsinterface.h"
  21 #include "third_party/webrtc/modules/audio_processing/typing_detection.h"
  22
  23 #if defined(OS_CHROMEOS)
  24 #include "base/sys_info.h"
  25 #endif
  26
  27 namespace content {
  28
  29 namespace {
  30
  31 using webrtc::AudioProcessing;
  32 using webrtc::NoiseSuppression;
  33
  34 const int kAudioProcessingNumberOfChannels = 1;
  35
  36 AudioProcessing::ChannelLayout MapLayout(media::ChannelLayout media_layout) {
  37   switch (media_layout) {
  38     case media::CHANNEL_LAYOUT_MONO:
  39       return AudioProcessing::kMono;
  40     case media::CHANNEL_LAYOUT_STEREO:
  41       return AudioProcessing::kStereo;
  42     case media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC:
  43       return AudioProcessing::kStereoAndKeyboard;
  44     default:
  45       NOTREACHED() << "Layout not supported: " << media_layout;
  46       return AudioProcessing::kMono;
  47   }
  48 }
  49
  50 // This is only used for playout data where only max two channels is supported.
  51 AudioProcessing::ChannelLayout ChannelsToLayout(int num_channels) {
  52   switch (num_channels) {
  53     case 1:
  54       return AudioProcessing::kMono;
  55     case 2:
  56       return AudioProcessing::kStereo;
  57     default:
  58       NOTREACHED() << "Channels not supported: " << num_channels;
  59       return AudioProcessing::kMono;
  60   }
  61 }
  62
  63 // Used by UMA histograms and entries shouldn't be re-ordered or removed.
  64 enum AudioTrackProcessingStates {
  65   AUDIO_PROCESSING_ENABLED = 0,
  66   AUDIO_PROCESSING_DISABLED,
  67   AUDIO_PROCESSING_IN_WEBRTC,
  68   AUDIO_PROCESSING_MAX
  69 };
  70
  71 void RecordProcessingState(AudioTrackProcessingStates state) {
  72   UMA_HISTOGRAM_ENUMERATION("Media.AudioTrackProcessingStates",
  73                             state, AUDIO_PROCESSING_MAX);
  74 }
  75
  76 bool IsDelayAgnosticAecEnabled() {
  77   // Note: It's important to query the field trial state first, to ensure that
  78   // UMA reports the correct group.
  79   const std::string group_name =
  80       base::FieldTrialList::FindFullName("UseDelayAgnosticAEC");
  81   base::CommandLine* command_line = base::CommandLine::ForCurrentProcess();
  82   if (command_line->HasSwitch(switches::kEnableDelayAgnosticAec))
  83     return true;
  84   if (command_line->HasSwitch(switches::kDisableDelayAgnosticAec))
  85     return false;
  86
  87   return (group_name == "Enabled" || group_name == "DefaultEnabled");
  88 }
  89
  90 bool IsBeamformingEnabled(const MediaAudioConstraints& audio_constraints) {
  91   return base::FieldTrialList::FindFullName("ChromebookBeamforming") ==
  92              "Enabled" ||
  93          audio_constraints.GetProperty(MediaAudioConstraints::kGoogBeamforming);
  94 }
  95
  96 }  // namespace
  97
  98 // Wraps AudioBus to provide access to the array of channel pointers, since this
  99 // is the type webrtc::AudioProcessing deals in. The array is refreshed on every
 100 // channel_ptrs() call, and will be valid until the underlying AudioBus pointers
 101 // are changed, e.g. through calls to SetChannelData() or SwapChannels().
 102 //
 103 // All methods are called on one of the capture or render audio threads
 104 // exclusively.
 105 class MediaStreamAudioBus {
 106  public:
 107   MediaStreamAudioBus(int channels, int frames)
 108       : bus_(media::AudioBus::Create(channels, frames)),
 109         channel_ptrs_(new float*[channels]) {
 110     // May be created in the main render thread and used in the audio threads.
 111     thread_checker_.DetachFromThread();
 112   }
 113
 114   media::AudioBus* bus() {
 115     DCHECK(thread_checker_.CalledOnValidThread());
 116     return bus_.get();
 117   }
 118
 119   float* const* channel_ptrs() {
 120     DCHECK(thread_checker_.CalledOnValidThread());
 121     for (int i = 0; i < bus_->channels(); ++i) {
 122       channel_ptrs_[i] = bus_->channel(i);
 123     }
 124     return channel_ptrs_.get();
 125   }
 126
 127  private:
 128   base::ThreadChecker thread_checker_;
 129   scoped_ptr<media::AudioBus> bus_;
 130   scoped_ptr<float*[]> channel_ptrs_;
 131 };
 132
 133 // Wraps AudioFifo to provide a cleaner interface to MediaStreamAudioProcessor.
 134 // It avoids the FIFO when the source and destination frames match. All methods
 135 // are called on one of the capture or render audio threads exclusively. If
 136 // |source_channels| is larger than |destination_channels|, only the first
 137 // |destination_channels| are kept from the source.
 138 class MediaStreamAudioFifo {
 139  public:
 140   MediaStreamAudioFifo(int source_channels,
 141                        int destination_channels,
 142                        int source_frames,
 143                        int destination_frames,
 144                        int sample_rate)
 145      : source_channels_(source_channels),
 146        source_frames_(source_frames),
 147        sample_rate_(sample_rate),
 148        destination_(
 149            new MediaStreamAudioBus(destination_channels, destination_frames)),
 150        data_available_(false) {
 151     DCHECK_GE(source_channels, destination_channels);
 152     DCHECK_GT(sample_rate_, 0);
 153
 154     if (source_channels > destination_channels) {
 155       audio_source_intermediate_ =
 156           media::AudioBus::CreateWrapper(destination_channels);
 157     }
 158
 159     if (source_frames != destination_frames) {
 160       // Since we require every Push to be followed by as many Consumes as
 161       // possible, twice the larger of the two is a (probably) loose upper bound
 162       // on the FIFO size.
 163       const int fifo_frames = 2 * std::max(source_frames, destination_frames);
 164       fifo_.reset(new media::AudioFifo(destination_channels, fifo_frames));
 165     }
 166
 167     // May be created in the main render thread and used in the audio threads.
 168     thread_checker_.DetachFromThread();
 169   }
 170
 171   void Push(const media::AudioBus& source, base::TimeDelta audio_delay) {
 172     DCHECK(thread_checker_.CalledOnValidThread());
 173     DCHECK_EQ(source.channels(), source_channels_);
 174     DCHECK_EQ(source.frames(), source_frames_);
 175
 176     const media::AudioBus* source_to_push = &source;
 177
 178     if (audio_source_intermediate_) {
 179       for (int i = 0; i < destination_->bus()->channels(); ++i) {
 180         audio_source_intermediate_->SetChannelData(
 181             i,
 182             const_cast<float*>(source.channel(i)));
 183       }
 184       audio_source_intermediate_->set_frames(source.frames());
 185       source_to_push = audio_source_intermediate_.get();
 186     }
 187
 188     if (fifo_) {
 189       next_audio_delay_ = audio_delay +
 190           fifo_->frames() * base::TimeDelta::FromSeconds(1) / sample_rate_;
 191       fifo_->Push(source_to_push);
 192     } else {
 193       source_to_push->CopyTo(destination_->bus());
 194       next_audio_delay_ = audio_delay;
 195       data_available_ = true;
 196     }
 197   }
 198
 199   // Returns true if there are destination_frames() of data available to be
 200   // consumed, and otherwise false.
 201   bool Consume(MediaStreamAudioBus** destination,
 202                base::TimeDelta* audio_delay) {
 203     DCHECK(thread_checker_.CalledOnValidThread());
 204
 205     if (fifo_) {
 206       if (fifo_->frames() < destination_->bus()->frames())
 207         return false;
 208
 209       fifo_->Consume(destination_->bus(), 0, destination_->bus()->frames());
 210       *audio_delay = next_audio_delay_;
 211       next_audio_delay_ -=
 212           destination_->bus()->frames() * base::TimeDelta::FromSeconds(1) /
 213               sample_rate_;
 214     } else {
 215       if (!data_available_)
 216         return false;
 217       *audio_delay = next_audio_delay_;
 218       // The data was already copied to |destination_| in this case.
 219       data_available_ = false;
 220     }
 221
 222     *destination = destination_.get();
 223     return true;
 224   }
 225
 226  private:
 227   base::ThreadChecker thread_checker_;
 228   const int source_channels_;  // For a DCHECK.
 229   const int source_frames_;  // For a DCHECK.
 230   const int sample_rate_;
 231   scoped_ptr<media::AudioBus> audio_source_intermediate_;
 232   scoped_ptr<MediaStreamAudioBus> destination_;
 233   scoped_ptr<media::AudioFifo> fifo_;
 234
 235   // When using |fifo_|, this is the audio delay of the first sample to be
 236   // consumed next from the FIFO.  When not using |fifo_|, this is the audio
 237   // delay of the first sample in |destination_|.
 238   base::TimeDelta next_audio_delay_;
 239
 240   // True when |destination_| contains the data to be returned by the next call
 241   // to Consume().  Only used when the FIFO is disabled.
 242   bool data_available_;
 243 };
 244
 245 MediaStreamAudioProcessor::MediaStreamAudioProcessor(
 246     const blink::WebMediaConstraints& constraints,
 247     int effects,
 248     WebRtcPlayoutDataSource* playout_data_source)
 249     : render_delay_ms_(0),
 250       playout_data_source_(playout_data_source),
 251       audio_mirroring_(false),
 252       typing_detected_(false),
 253       stopped_(false) {
 254   capture_thread_checker_.DetachFromThread();
 255   render_thread_checker_.DetachFromThread();
 256   InitializeAudioProcessingModule(constraints, effects);
 257
 258   aec_dump_message_filter_ = AecDumpMessageFilter::Get();
 259   // In unit tests not creating a message filter, |aec_dump_message_filter_|
 260   // will be NULL. We can just ignore that. Other unit tests and browser tests
 261   // ensure that we do get the filter when we should.
 262   if (aec_dump_message_filter_.get())
 263     aec_dump_message_filter_->AddDelegate(this);
 264 }
 265
 266 MediaStreamAudioProcessor::~MediaStreamAudioProcessor() {
 267   DCHECK(main_thread_checker_.CalledOnValidThread());
 268   Stop();
 269 }
 270
 271 void MediaStreamAudioProcessor::OnCaptureFormatChanged(
 272     const media::AudioParameters& input_format) {
 273   DCHECK(main_thread_checker_.CalledOnValidThread());
 274   // There is no need to hold a lock here since the caller guarantees that
 275   // there is no more PushCaptureData() and ProcessAndConsumeData() callbacks
 276   // on the capture thread.
 277   InitializeCaptureFifo(input_format);
 278
 279   // Reset the |capture_thread_checker_| since the capture data will come from
 280   // a new capture thread.
 281   capture_thread_checker_.DetachFromThread();
 282 }
 283
 284 void MediaStreamAudioProcessor::PushCaptureData(
 285     const media::AudioBus& audio_source,
 286     base::TimeDelta capture_delay) {
 287   DCHECK(capture_thread_checker_.CalledOnValidThread());
 288
 289   capture_fifo_->Push(audio_source, capture_delay);
 290 }
 291
 292 bool MediaStreamAudioProcessor::ProcessAndConsumeData(
 293     int volume,
 294     bool key_pressed,
 295     media::AudioBus** processed_data,
 296     base::TimeDelta* capture_delay,
 297     int* new_volume) {
 298   DCHECK(capture_thread_checker_.CalledOnValidThread());
 299   DCHECK(processed_data);
 300   DCHECK(capture_delay);
 301   DCHECK(new_volume);
 302
 303   TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessAndConsumeData");
 304
 305   MediaStreamAudioBus* process_bus;
 306   if (!capture_fifo_->Consume(&process_bus, capture_delay))
 307     return false;
 308
 309   // Use the process bus directly if audio processing is disabled.
 310   MediaStreamAudioBus* output_bus = process_bus;
 311   *new_volume = 0;
 312   if (audio_processing_) {
 313     output_bus = output_bus_.get();
 314     *new_volume = ProcessData(process_bus->channel_ptrs(),
 315                               process_bus->bus()->frames(), *capture_delay,
 316                               volume, key_pressed, output_bus->channel_ptrs());
 317   }
 318
 319   // Swap channels before interleaving the data.
 320   if (audio_mirroring_ &&
 321       output_format_.channel_layout() == media::CHANNEL_LAYOUT_STEREO) {
 322     // Swap the first and second channels.
 323     output_bus->bus()->SwapChannels(0, 1);
 324   }
 325
 326   *processed_data = output_bus->bus();
 327
 328   return true;
 329 }
 330
 331 void MediaStreamAudioProcessor::Stop() {
 332   DCHECK(main_thread_checker_.CalledOnValidThread());
 333   if (stopped_)
 334     return;
 335
 336   stopped_ = true;
 337
 338   if (aec_dump_message_filter_.get()) {
 339     aec_dump_message_filter_->RemoveDelegate(this);
 340     aec_dump_message_filter_ = NULL;
 341   }
 342
 343   if (!audio_processing_.get())
 344     return;
 345
 346   audio_processing_.get()->UpdateHistogramsOnCallEnd();
 347   StopEchoCancellationDump(audio_processing_.get());
 348
 349   if (playout_data_source_) {
 350     playout_data_source_->RemovePlayoutSink(this);
 351     playout_data_source_ = NULL;
 352   }
 353 }
 354
 355 const media::AudioParameters& MediaStreamAudioProcessor::InputFormat() const {
 356   return input_format_;
 357 }
 358
 359 const media::AudioParameters& MediaStreamAudioProcessor::OutputFormat() const {
 360   return output_format_;
 361 }
 362
 363 void MediaStreamAudioProcessor::OnAecDumpFile(
 364     const IPC::PlatformFileForTransit& file_handle) {
 365   DCHECK(main_thread_checker_.CalledOnValidThread());
 366
 367   base::File file = IPC::PlatformFileForTransitToFile(file_handle);
 368   DCHECK(file.IsValid());
 369
 370   if (audio_processing_)
 371     StartEchoCancellationDump(audio_processing_.get(), file.Pass());
 372   else
 373     file.Close();
 374 }
 375
 376 void MediaStreamAudioProcessor::OnDisableAecDump() {
 377   DCHECK(main_thread_checker_.CalledOnValidThread());
 378   if (audio_processing_)
 379     StopEchoCancellationDump(audio_processing_.get());
 380 }
 381
 382 void MediaStreamAudioProcessor::OnIpcClosing() {
 383   DCHECK(main_thread_checker_.CalledOnValidThread());
 384   aec_dump_message_filter_ = NULL;
 385 }
 386
 387 void MediaStreamAudioProcessor::OnPlayoutData(media::AudioBus* audio_bus,
 388                                               int sample_rate,
 389                                               int audio_delay_milliseconds) {
 390   DCHECK(render_thread_checker_.CalledOnValidThread());
 391   DCHECK(audio_processing_->echo_control_mobile()->is_enabled() ^
 392          audio_processing_->echo_cancellation()->is_enabled());
 393
 394   TRACE_EVENT0("audio", "MediaStreamAudioProcessor::OnPlayoutData");
 395   DCHECK_LT(audio_delay_milliseconds,
 396             std::numeric_limits<base::subtle::Atomic32>::max());
 397   base::subtle::Release_Store(&render_delay_ms_, audio_delay_milliseconds);
 398
 399   InitializeRenderFifoIfNeeded(sample_rate, audio_bus->channels(),
 400                                audio_bus->frames());
 401
 402   render_fifo_->Push(
 403       *audio_bus, base::TimeDelta::FromMilliseconds(audio_delay_milliseconds));
 404   MediaStreamAudioBus* analysis_bus;
 405   base::TimeDelta audio_delay;
 406   while (render_fifo_->Consume(&analysis_bus, &audio_delay)) {
 407     // TODO(ajm): Should AnalyzeReverseStream() account for the |audio_delay|?
 408     audio_processing_->AnalyzeReverseStream(
 409         analysis_bus->channel_ptrs(),
 410         analysis_bus->bus()->frames(),
 411         sample_rate,
 412         ChannelsToLayout(audio_bus->channels()));
 413   }
 414 }
 415
 416 void MediaStreamAudioProcessor::OnPlayoutDataSourceChanged() {
 417   DCHECK(main_thread_checker_.CalledOnValidThread());
 418   // There is no need to hold a lock here since the caller guarantees that
 419   // there is no more OnPlayoutData() callback on the render thread.
 420   render_thread_checker_.DetachFromThread();
 421   render_fifo_.reset();
 422 }
 423
 424 void MediaStreamAudioProcessor::GetStats(AudioProcessorStats* stats) {
 425   stats->typing_noise_detected =
 426       (base::subtle::Acquire_Load(&typing_detected_) != false);
 427   GetAecStats(audio_processing_.get()->echo_cancellation(), stats);
 428 }
 429
 430 void MediaStreamAudioProcessor::InitializeAudioProcessingModule(
 431     const blink::WebMediaConstraints& constraints, int effects) {
 432   DCHECK(main_thread_checker_.CalledOnValidThread());
 433   DCHECK(!audio_processing_);
 434
 435   MediaAudioConstraints audio_constraints(constraints, effects);
 436
 437   // Audio mirroring can be enabled even though audio processing is otherwise
 438   // disabled.
 439   audio_mirroring_ = audio_constraints.GetProperty(
 440       MediaAudioConstraints::kGoogAudioMirroring);
 441
 442 #if defined(OS_IOS)
 443   // On iOS, VPIO provides built-in AGC and AEC.
 444   const bool echo_cancellation = false;
 445   const bool goog_agc = false;
 446 #else
 447   const bool echo_cancellation =
 448       audio_constraints.GetEchoCancellationProperty();
 449   const bool goog_agc = audio_constraints.GetProperty(
 450       MediaAudioConstraints::kGoogAutoGainControl);
 451 #endif
 452
 453 #if defined(OS_IOS) || defined(OS_ANDROID)
 454   const bool goog_experimental_aec = false;
 455   const bool goog_typing_detection = false;
 456 #else
 457   const bool goog_experimental_aec = audio_constraints.GetProperty(
 458       MediaAudioConstraints::kGoogExperimentalEchoCancellation);
 459   const bool goog_typing_detection = audio_constraints.GetProperty(
 460       MediaAudioConstraints::kGoogTypingNoiseDetection);
 461 #endif
 462
 463   const bool goog_ns = audio_constraints.GetProperty(
 464       MediaAudioConstraints::kGoogNoiseSuppression);
 465   const bool goog_experimental_ns = audio_constraints.GetProperty(
 466       MediaAudioConstraints::kGoogExperimentalNoiseSuppression);
 467   const bool goog_beamforming = IsBeamformingEnabled(audio_constraints);
 468   const bool goog_high_pass_filter = audio_constraints.GetProperty(
 469       MediaAudioConstraints::kGoogHighpassFilter);
 470   // Return immediately if no goog constraint is enabled.
 471   if (!echo_cancellation && !goog_experimental_aec && !goog_ns &&
 472       !goog_high_pass_filter && !goog_typing_detection &&
 473       !goog_agc && !goog_experimental_ns && !goog_beamforming) {
 474     RecordProcessingState(AUDIO_PROCESSING_DISABLED);
 475     return;
 476   }
 477
 478   // Experimental options provided at creation.
 479   webrtc::Config config;
 480   if (goog_experimental_aec)
 481     config.Set<webrtc::ExtendedFilter>(new webrtc::ExtendedFilter(true));
 482   if (goog_experimental_ns)
 483     config.Set<webrtc::ExperimentalNs>(new webrtc::ExperimentalNs(true));
 484   if (IsDelayAgnosticAecEnabled())
 485     config.Set<webrtc::DelayAgnostic>(new webrtc::DelayAgnostic(true));
 486   if (goog_beamforming) {
 487     ConfigureBeamforming(&config, audio_constraints.GetPropertyAsString(
 488         MediaAudioConstraints::kGoogArrayGeometry));
 489   }
 490
 491   // Create and configure the webrtc::AudioProcessing.
 492   audio_processing_.reset(webrtc::AudioProcessing::Create(config));
 493
 494   // Enable the audio processing components.
 495   if (echo_cancellation) {
 496     EnableEchoCancellation(audio_processing_.get());
 497
 498     if (playout_data_source_)
 499       playout_data_source_->AddPlayoutSink(this);
 500
 501     // Prepare for logging echo information. If there are data remaining in
 502     // |echo_information_| we simply discard it.
 503     echo_information_.reset(new EchoInformation());
 504   }
 505
 506   if (goog_ns) {
 507     // The beamforming postfilter is effective at suppressing stationary noise,
 508     // so reduce the single-channel NS aggressiveness when enabled.
 509     const NoiseSuppression::Level ns_level =
 510         config.Get<webrtc::Beamforming>().enabled ? NoiseSuppression::kLow
 511                                                   : NoiseSuppression::kHigh;
 512
 513     EnableNoiseSuppression(audio_processing_.get(), ns_level);
 514   }
 515
 516   if (goog_high_pass_filter)
 517     EnableHighPassFilter(audio_processing_.get());
 518
 519   if (goog_typing_detection) {
 520     // TODO(xians): Remove this |typing_detector_| after the typing suppression
 521     // is enabled by default.
 522     typing_detector_.reset(new webrtc::TypingDetection());
 523     EnableTypingDetection(audio_processing_.get(), typing_detector_.get());
 524   }
 525
 526   if (goog_agc)
 527     EnableAutomaticGainControl(audio_processing_.get());
 528
 529   RecordProcessingState(AUDIO_PROCESSING_ENABLED);
 530 }
 531
 532 void MediaStreamAudioProcessor::ConfigureBeamforming(
 533     webrtc::Config* config,
 534     const std::string& geometry_str) const {
 535   std::vector<webrtc::Point> geometry = ParseArrayGeometry(geometry_str);
 536 #if defined(OS_CHROMEOS)
 537   if (geometry.size() == 0) {
 538     const std::string board = base::SysInfo::GetLsbReleaseBoard();
 539     if (board.find("nyan_kitty") != std::string::npos) {
 540       geometry.push_back(webrtc::Point(-0.03f, 0.f, 0.f));
 541       geometry.push_back(webrtc::Point(0.03f, 0.f, 0.f));
 542     } else if (board.find("peach_pi") != std::string::npos) {
 543       geometry.push_back(webrtc::Point(-0.025f, 0.f, 0.f));
 544       geometry.push_back(webrtc::Point(0.025f, 0.f, 0.f));
 545     } else if (board.find("samus") != std::string::npos) {
 546       geometry.push_back(webrtc::Point(-0.032f, 0.f, 0.f));
 547       geometry.push_back(webrtc::Point(0.032f, 0.f, 0.f));
 548     } else if (board.find("swanky") != std::string::npos) {
 549       geometry.push_back(webrtc::Point(-0.026f, 0.f, 0.f));
 550       geometry.push_back(webrtc::Point(0.026f, 0.f, 0.f));
 551     }
 552   }
 553 #endif
 554   config->Set<webrtc::Beamforming>(new webrtc::Beamforming(geometry.size() > 1,
 555                                                            geometry));
 556 }
 557
 558 std::vector<webrtc::Point> MediaStreamAudioProcessor::ParseArrayGeometry(
 559     const std::string& geometry_str) const {
 560   std::vector<webrtc::Point> result;
 561   std::vector<float> values;
 562   std::istringstream str(geometry_str);
 563   std::copy(std::istream_iterator<float>(str),
 564             std::istream_iterator<float>(),
 565             std::back_inserter(values));
 566   if (values.size() % 3 == 0) {
 567     for (size_t i = 0; i < values.size(); i += 3) {
 568       result.push_back(webrtc::Point(values[i + 0],
 569                                      values[i + 1],
 570                                      values[i + 2]));
 571     }
 572   }
 573   return result;
 574 }
 575
 576 void MediaStreamAudioProcessor::InitializeCaptureFifo(
 577     const media::AudioParameters& input_format) {
 578   DCHECK(main_thread_checker_.CalledOnValidThread());
 579   DCHECK(input_format.IsValid());
 580   input_format_ = input_format;
 581
 582   // TODO(ajm): For now, we assume fixed parameters for the output when audio
 583   // processing is enabled, to match the previous behavior. We should either
 584   // use the input parameters (in which case, audio processing will convert
 585   // at output) or ideally, have a backchannel from the sink to know what
 586   // format it would prefer.
 587 #if defined(OS_ANDROID)
 588   int audio_processing_sample_rate = AudioProcessing::kSampleRate16kHz;
 589 #else
 590   int audio_processing_sample_rate = AudioProcessing::kSampleRate48kHz;
 591 #endif
 592   const int output_sample_rate = audio_processing_ ?
 593                                  audio_processing_sample_rate :
 594                                  input_format.sample_rate();
 595   media::ChannelLayout output_channel_layout = audio_processing_ ?
 596       media::GuessChannelLayout(kAudioProcessingNumberOfChannels) :
 597       input_format.channel_layout();
 598
 599   // The output channels from the fifo is normally the same as input.
 600   int fifo_output_channels = input_format.channels();
 601
 602   // Special case for if we have a keyboard mic channel on the input and no
 603   // audio processing is used. We will then have the fifo strip away that
 604   // channel. So we use stereo as output layout, and also change the output
 605   // channels for the fifo.
 606   if (input_format.channel_layout() ==
 607           media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC &&
 608       !audio_processing_) {
 609     output_channel_layout = media::CHANNEL_LAYOUT_STEREO;
 610     fifo_output_channels = ChannelLayoutToChannelCount(output_channel_layout);
 611   }
 612
 613   // webrtc::AudioProcessing requires a 10 ms chunk size. We use this native
 614   // size when processing is enabled. When disabled we use the same size as
 615   // the source if less than 10 ms.
 616   //
 617   // TODO(ajm): This conditional buffer size appears to be assuming knowledge of
 618   // the sink based on the source parameters. PeerConnection sinks seem to want
 619   // 10 ms chunks regardless, while WebAudio sinks want less, and we're assuming
 620   // we can identify WebAudio sinks by the input chunk size. Less fragile would
 621   // be to have the sink actually tell us how much it wants (as in the above
 622   // TODO).
 623   int processing_frames = input_format.sample_rate() / 100;
 624   int output_frames = output_sample_rate / 100;
 625   if (!audio_processing_ && input_format.frames_per_buffer() < output_frames) {
 626     processing_frames = input_format.frames_per_buffer();
 627     output_frames = processing_frames;
 628   }
 629
 630   output_format_ = media::AudioParameters(
 631       media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 632       output_channel_layout,
 633       output_sample_rate,
 634       16,
 635       output_frames);
 636
 637   capture_fifo_.reset(
 638       new MediaStreamAudioFifo(input_format.channels(),
 639                                fifo_output_channels,
 640                                input_format.frames_per_buffer(),
 641                                processing_frames,
 642                                input_format.sample_rate()));
 643
 644   if (audio_processing_) {
 645     output_bus_.reset(new MediaStreamAudioBus(output_format_.channels(),
 646                                               output_frames));
 647   }
 648 }
 649
 650 void MediaStreamAudioProcessor::InitializeRenderFifoIfNeeded(
 651     int sample_rate, int number_of_channels, int frames_per_buffer) {
 652   DCHECK(render_thread_checker_.CalledOnValidThread());
 653   if (render_fifo_.get() &&
 654       render_format_.sample_rate() == sample_rate &&
 655       render_format_.channels() == number_of_channels &&
 656       render_format_.frames_per_buffer() == frames_per_buffer) {
 657     // Do nothing if the |render_fifo_| has been setup properly.
 658     return;
 659   }
 660
 661   render_format_ = media::AudioParameters(
 662       media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
 663       media::GuessChannelLayout(number_of_channels),
 664       sample_rate,
 665       16,
 666       frames_per_buffer);
 667
 668   const int analysis_frames = sample_rate / 100;  // 10 ms chunks.
 669   render_fifo_.reset(
 670       new MediaStreamAudioFifo(number_of_channels,
 671                                number_of_channels,
 672                                frames_per_buffer,
 673                                analysis_frames,
 674                                sample_rate));
 675 }
 676
 677 int MediaStreamAudioProcessor::ProcessData(const float* const* process_ptrs,
 678                                            int process_frames,
 679                                            base::TimeDelta capture_delay,
 680                                            int volume,
 681                                            bool key_pressed,
 682                                            float* const* output_ptrs) {
 683   DCHECK(audio_processing_);
 684   DCHECK(capture_thread_checker_.CalledOnValidThread());
 685
 686   TRACE_EVENT0("audio", "MediaStreamAudioProcessor::ProcessData");
 687
 688   base::subtle::Atomic32 render_delay_ms =
 689       base::subtle::Acquire_Load(&render_delay_ms_);
 690   int64 capture_delay_ms = capture_delay.InMilliseconds();
 691   DCHECK_LT(capture_delay_ms,
 692             std::numeric_limits<base::subtle::Atomic32>::max());
 693   int total_delay_ms =  capture_delay_ms + render_delay_ms;
 694   if (total_delay_ms > 300) {
 695     LOG(WARNING) << "Large audio delay, capture delay: " << capture_delay_ms
 696                  << "ms; render delay: " << render_delay_ms << "ms";
 697   }
 698
 699   webrtc::AudioProcessing* ap = audio_processing_.get();
 700   ap->set_stream_delay_ms(total_delay_ms);
 701
 702   DCHECK_LE(volume, WebRtcAudioDeviceImpl::kMaxVolumeLevel);
 703   webrtc::GainControl* agc = ap->gain_control();
 704   int err = agc->set_stream_analog_level(volume);
 705   DCHECK_EQ(err, 0) << "set_stream_analog_level() error: " << err;
 706
 707   ap->set_stream_key_pressed(key_pressed);
 708
 709   err = ap->ProcessStream(process_ptrs,
 710                           process_frames,
 711                           input_format_.sample_rate(),
 712                           MapLayout(input_format_.channel_layout()),
 713                           output_format_.sample_rate(),
 714                           MapLayout(output_format_.channel_layout()),
 715                           output_ptrs);
 716   DCHECK_EQ(err, 0) << "ProcessStream() error: " << err;
 717
 718   if (typing_detector_) {
 719     webrtc::VoiceDetection* vad = ap->voice_detection();
 720     DCHECK(vad->is_enabled());
 721     bool detected = typing_detector_->Process(key_pressed,
 722                                               vad->stream_has_voice());
 723     base::subtle::Release_Store(&typing_detected_, detected);
 724   }
 725
 726   if (echo_information_) {
 727     echo_information_.get()->UpdateAecDelayStats(ap->echo_cancellation());
 728   }
 729
 730   // Return 0 if the volume hasn't been changed, and otherwise the new volume.
 731   return (agc->stream_analog_level() == volume) ?
 732       0 : agc->stream_analog_level();
 733 }
 734
 735 }  // namespace content