content/renderer/media/media_stream_audio_processor_options.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/media_stream_audio_processor_options.h"
   6
   7 #include "base/files/file_path.h"
   8 #include "base/files/file_util.h"
   9 #include "base/logging.h"
  10 #include "base/metrics/field_trial.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/strings/string_number_conversions.h"
  13 #include "base/strings/utf_string_conversions.h"
  14 #include "content/common/media/media_stream_options.h"
  15 #include "content/renderer/media/media_stream_constraints_util.h"
  16 #include "content/renderer/media/media_stream_source.h"
  17 #include "content/renderer/media/rtc_media_constraints.h"
  18 #include "media/audio/audio_parameters.h"
  19 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
  20 #include "third_party/webrtc/modules/audio_processing/typing_detection.h"
  21
  22 namespace content {
  23
  24 const char MediaAudioConstraints::kEchoCancellation[] = "echoCancellation";
  25 const char MediaAudioConstraints::kGoogEchoCancellation[] =
  26     "googEchoCancellation";
  27 const char MediaAudioConstraints::kGoogExperimentalEchoCancellation[] =
  28     "googEchoCancellation2";
  29 const char MediaAudioConstraints::kGoogAutoGainControl[] =
  30     "googAutoGainControl";
  31 const char MediaAudioConstraints::kGoogExperimentalAutoGainControl[] =
  32     "googAutoGainControl2";
  33 const char MediaAudioConstraints::kGoogNoiseSuppression[] =
  34     "googNoiseSuppression";
  35 const char MediaAudioConstraints::kGoogExperimentalNoiseSuppression[] =
  36     "googNoiseSuppression2";
  37 const char MediaAudioConstraints::kGoogBeamforming[] = "googBeamforming";
  38 const char MediaAudioConstraints::kGoogArrayGeometry[] = "googArrayGeometry";
  39 const char MediaAudioConstraints::kGoogHighpassFilter[] = "googHighpassFilter";
  40 const char MediaAudioConstraints::kGoogTypingNoiseDetection[] =
  41     "googTypingNoiseDetection";
  42 const char MediaAudioConstraints::kGoogAudioMirroring[] = "googAudioMirroring";
  43
  44 namespace {
  45
  46 // Constant constraint keys which enables default audio constraints on
  47 // mediastreams with audio.
  48 struct {
  49   const char* key;
  50   bool value;
  51 } const kDefaultAudioConstraints[] = {
  52   { MediaAudioConstraints::kEchoCancellation, true },
  53   { MediaAudioConstraints::kGoogEchoCancellation, true },
  54 #if defined(OS_ANDROID) || defined(OS_IOS)
  55   { MediaAudioConstraints::kGoogExperimentalEchoCancellation, false },
  56 #else
  57   // Enable the extended filter mode AEC on all non-mobile platforms.
  58   { MediaAudioConstraints::kGoogExperimentalEchoCancellation, true },
  59 #endif
  60   { MediaAudioConstraints::kGoogAutoGainControl, true },
  61   { MediaAudioConstraints::kGoogExperimentalAutoGainControl, true },
  62   { MediaAudioConstraints::kGoogNoiseSuppression, true },
  63   { MediaAudioConstraints::kGoogHighpassFilter, true },
  64   { MediaAudioConstraints::kGoogTypingNoiseDetection, true },
  65   { MediaAudioConstraints::kGoogExperimentalNoiseSuppression, false },
  66   { MediaAudioConstraints::kGoogBeamforming, false },
  67 #if defined(OS_WIN)
  68   { kMediaStreamAudioDucking, true },
  69 #else
  70   { kMediaStreamAudioDucking, false },
  71 #endif
  72   { kMediaStreamAudioHotword, false },
  73 };
  74
  75 bool IsAudioProcessingConstraint(const std::string& key) {
  76   // |kMediaStreamAudioDucking| does not require audio processing.
  77   return key != kMediaStreamAudioDucking;
  78 }
  79
  80 // Used to log echo quality based on delay estimates.
  81 enum DelayBasedEchoQuality {
  82   DELAY_BASED_ECHO_QUALITY_GOOD = 0,
  83   DELAY_BASED_ECHO_QUALITY_SPURIOUS,
  84   DELAY_BASED_ECHO_QUALITY_BAD,
  85   DELAY_BASED_ECHO_QUALITY_INVALID,
  86   DELAY_BASED_ECHO_QUALITY_MAX
  87 };
  88
  89 DelayBasedEchoQuality EchoDelayFrequencyToQuality(float delay_frequency) {
  90   const float kEchoDelayFrequencyLowerLimit = 0.1f;
  91   const float kEchoDelayFrequencyUpperLimit = 0.8f;
  92   // DELAY_BASED_ECHO_QUALITY_GOOD
  93   //   delay is out of bounds during at most 10 % of the time.
  94   // DELAY_BASED_ECHO_QUALITY_SPURIOUS
  95   //   delay is out of bounds 10-80 % of the time.
  96   // DELAY_BASED_ECHO_QUALITY_BAD
  97   //   delay is mostly out of bounds >= 80 % of the time.
  98   // DELAY_BASED_ECHO_QUALITY_INVALID
  99   //   delay_frequency is negative which happens if we have insufficient data.
 100   if (delay_frequency < 0)
 101     return DELAY_BASED_ECHO_QUALITY_INVALID;
 102   else if (delay_frequency <= kEchoDelayFrequencyLowerLimit)
 103     return DELAY_BASED_ECHO_QUALITY_GOOD;
 104   else if (delay_frequency < kEchoDelayFrequencyUpperLimit)
 105     return DELAY_BASED_ECHO_QUALITY_SPURIOUS;
 106   else
 107     return DELAY_BASED_ECHO_QUALITY_BAD;
 108 }
 109
 110 }  // namespace
 111
 112 // TODO(xians): Remove this method after the APM in WebRtc is deprecated.
 113 void MediaAudioConstraints::ApplyFixedAudioConstraints(
 114     RTCMediaConstraints* constraints) {
 115   for (size_t i = 0; i < arraysize(kDefaultAudioConstraints); ++i) {
 116     bool already_set_value;
 117     if (!webrtc::FindConstraint(constraints, kDefaultAudioConstraints[i].key,
 118                                 &already_set_value, NULL)) {
 119       const std::string value = kDefaultAudioConstraints[i].value ?
 120           webrtc::MediaConstraintsInterface::kValueTrue :
 121           webrtc::MediaConstraintsInterface::kValueFalse;
 122       constraints->AddOptional(kDefaultAudioConstraints[i].key, value, false);
 123     } else {
 124       DVLOG(1) << "Constraint " << kDefaultAudioConstraints[i].key
 125                << " already set to " << already_set_value;
 126     }
 127   }
 128 }
 129
 130 MediaAudioConstraints::MediaAudioConstraints(
 131     const blink::WebMediaConstraints& constraints, int effects)
 132     : constraints_(constraints),
 133       effects_(effects),
 134       default_audio_processing_constraint_value_(true) {
 135   // The default audio processing constraints are turned off when
 136   // - gUM has a specific kMediaStreamSource, which is used by tab capture
 137   //   and screen capture.
 138   // - |kEchoCancellation| is explicitly set to false.
 139   std::string value_str;
 140   bool value_bool = false;
 141   if ((GetConstraintValueAsString(constraints, kMediaStreamSource,
 142                                   &value_str)) ||
 143       (GetConstraintValueAsBoolean(constraints_, kEchoCancellation,
 144                                    &value_bool) && !value_bool)) {
 145     default_audio_processing_constraint_value_ = false;
 146   }
 147 }
 148
 149 MediaAudioConstraints::~MediaAudioConstraints() {}
 150
 151 bool MediaAudioConstraints::GetProperty(const std::string& key) const {
 152   // Return the value if the constraint is specified in |constraints|,
 153   // otherwise return the default value.
 154   bool value = false;
 155   if (!GetConstraintValueAsBoolean(constraints_, key, &value))
 156     value = GetDefaultValueForConstraint(constraints_, key);
 157
 158   return value;
 159 }
 160
 161 std::string MediaAudioConstraints::GetPropertyAsString(
 162     const std::string& key) const {
 163   std::string value;
 164   GetConstraintValueAsString(constraints_, key, &value);
 165   return value;
 166 }
 167
 168 bool MediaAudioConstraints::GetEchoCancellationProperty() const {
 169   // If platform echo canceller is enabled, disable the software AEC.
 170   if (effects_ & media::AudioParameters::ECHO_CANCELLER)
 171     return false;
 172
 173   // If |kEchoCancellation| is specified in the constraints, it will
 174   // override the value of |kGoogEchoCancellation|.
 175   bool value = false;
 176   if (GetConstraintValueAsBoolean(constraints_, kEchoCancellation, &value))
 177     return value;
 178
 179   return GetProperty(kGoogEchoCancellation);
 180 }
 181
 182 bool MediaAudioConstraints::IsValid() const {
 183   blink::WebVector<blink::WebMediaConstraint> mandatory;
 184   constraints_.getMandatoryConstraints(mandatory);
 185   for (size_t i = 0; i < mandatory.size(); ++i) {
 186     const std::string key = mandatory[i].m_name.utf8();
 187     if (key == kMediaStreamSource || key == kMediaStreamSourceId ||
 188         key == MediaStreamSource::kSourceId) {
 189       // Ignore Chrome specific Tab capture and |kSourceId| constraints.
 190       continue;
 191     }
 192
 193     bool valid = false;
 194     for (size_t j = 0; j < arraysize(kDefaultAudioConstraints); ++j) {
 195       if (key == kDefaultAudioConstraints[j].key) {
 196         bool value = false;
 197         valid = GetMandatoryConstraintValueAsBoolean(constraints_, key, &value);
 198         break;
 199       }
 200     }
 201
 202     if (!valid) {
 203       DLOG(ERROR) << "Invalid MediaStream constraint. Name: " << key;
 204       return false;
 205     }
 206   }
 207
 208   return true;
 209 }
 210
 211 bool MediaAudioConstraints::GetDefaultValueForConstraint(
 212     const blink::WebMediaConstraints& constraints,
 213     const std::string& key) const {
 214   // |kMediaStreamAudioDucking| is not restricted by
 215   // |default_audio_processing_constraint_value_| since it does not require
 216   // audio processing.
 217   if (!default_audio_processing_constraint_value_ &&
 218       IsAudioProcessingConstraint(key))
 219     return false;
 220
 221   for (size_t i = 0; i < arraysize(kDefaultAudioConstraints); ++i) {
 222     if (kDefaultAudioConstraints[i].key == key)
 223       return kDefaultAudioConstraints[i].value;
 224   }
 225
 226   return false;
 227 }
 228
 229 EchoInformation::EchoInformation()
 230     : num_chunks_(0), echo_frames_received_(false) {
 231 }
 232
 233 EchoInformation::~EchoInformation() {}
 234
 235 void EchoInformation::UpdateAecDelayStats(
 236     webrtc::EchoCancellation* echo_cancellation) {
 237   // Only start collecting stats if we know echo cancellation has measured an
 238   // echo. Otherwise we clutter the stats with for example cases where only the
 239   // microphone is used.
 240   if (!echo_frames_received_ & !echo_cancellation->stream_has_echo())
 241     return;
 242
 243   echo_frames_received_ = true;
 244   // In WebRTC, three echo delay metrics are calculated and updated every
 245   // five seconds. We use one of them, |fraction_poor_delays| to log in a UMA
 246   // histogram an Echo Cancellation quality metric. The stat in WebRTC has a
 247   // fixed aggregation window of five seconds, so we use the same query
 248   // frequency to avoid logging old values.
 249   const int kNumChunksInFiveSeconds = 500;
 250   if (!echo_cancellation->is_delay_logging_enabled() ||
 251       !echo_cancellation->is_enabled()) {
 252     return;
 253   }
 254
 255   num_chunks_++;
 256   if (num_chunks_ < kNumChunksInFiveSeconds) {
 257     return;
 258   }
 259
 260   int dummy_median = 0, dummy_std = 0;
 261   float fraction_poor_delays = 0;
 262   if (echo_cancellation->GetDelayMetrics(
 263           &dummy_median, &dummy_std, &fraction_poor_delays) ==
 264       webrtc::AudioProcessing::kNoError) {
 265     num_chunks_ = 0;
 266     // Map |fraction_poor_delays| to an Echo Cancellation quality and log in UMA
 267     // histogram. See DelayBasedEchoQuality for information on histogram
 268     // buckets.
 269     UMA_HISTOGRAM_ENUMERATION("WebRTC.AecDelayBasedQuality",
 270                               EchoDelayFrequencyToQuality(fraction_poor_delays),
 271                               DELAY_BASED_ECHO_QUALITY_MAX);
 272   }
 273 }
 274
 275 void EnableEchoCancellation(AudioProcessing* audio_processing) {
 276 #if defined(OS_ANDROID) || defined(OS_IOS)
 277   const std::string group_name =
 278       base::FieldTrialList::FindFullName("ReplaceAECMWithAEC");
 279   if (group_name.empty() ||
 280       !(group_name == "Enabled" || group_name == "DefaultEnabled")) {
 281     // Mobile devices are using AECM.
 282     int err = audio_processing->echo_control_mobile()->set_routing_mode(
 283         webrtc::EchoControlMobile::kSpeakerphone);
 284     err |= audio_processing->echo_control_mobile()->Enable(true);
 285     CHECK_EQ(err, 0);
 286     return;
 287   }
 288 #endif
 289   int err = audio_processing->echo_cancellation()->set_suppression_level(
 290       webrtc::EchoCancellation::kHighSuppression);
 291
 292   // Enable the metrics for AEC.
 293   err |= audio_processing->echo_cancellation()->enable_metrics(true);
 294   err |= audio_processing->echo_cancellation()->enable_delay_logging(true);
 295   err |= audio_processing->echo_cancellation()->Enable(true);
 296   CHECK_EQ(err, 0);
 297 }
 298
 299 void EnableNoiseSuppression(AudioProcessing* audio_processing,
 300                             webrtc::NoiseSuppression::Level ns_level) {
 301   int err = audio_processing->noise_suppression()->set_level(ns_level);
 302   err |= audio_processing->noise_suppression()->Enable(true);
 303   CHECK_EQ(err, 0);
 304 }
 305
 306 void EnableHighPassFilter(AudioProcessing* audio_processing) {
 307   CHECK_EQ(audio_processing->high_pass_filter()->Enable(true), 0);
 308 }
 309
 310 void EnableTypingDetection(AudioProcessing* audio_processing,
 311                            webrtc::TypingDetection* typing_detector) {
 312   int err = audio_processing->voice_detection()->Enable(true);
 313   err |= audio_processing->voice_detection()->set_likelihood(
 314       webrtc::VoiceDetection::kVeryLowLikelihood);
 315   CHECK_EQ(err, 0);
 316
 317   // Configure the update period to 1s (100 * 10ms) in the typing detector.
 318   typing_detector->SetParameters(0, 0, 0, 0, 0, 100);
 319 }
 320
 321 void StartEchoCancellationDump(AudioProcessing* audio_processing,
 322                                base::File aec_dump_file) {
 323   DCHECK(aec_dump_file.IsValid());
 324
 325   FILE* stream = base::FileToFILE(aec_dump_file.Pass(), "w");
 326   if (!stream) {
 327     LOG(ERROR) << "Failed to open AEC dump file";
 328     return;
 329   }
 330
 331   if (audio_processing->StartDebugRecording(stream))
 332     DLOG(ERROR) << "Fail to start AEC debug recording";
 333 }
 334
 335 void StopEchoCancellationDump(AudioProcessing* audio_processing) {
 336   if (audio_processing->StopDebugRecording())
 337     DLOG(ERROR) << "Fail to stop AEC debug recording";
 338 }
 339
 340 void EnableAutomaticGainControl(AudioProcessing* audio_processing) {
 341 #if defined(OS_ANDROID) || defined(OS_IOS)
 342   const webrtc::GainControl::Mode mode = webrtc::GainControl::kFixedDigital;
 343 #else
 344   const webrtc::GainControl::Mode mode = webrtc::GainControl::kAdaptiveAnalog;
 345 #endif
 346   int err = audio_processing->gain_control()->set_mode(mode);
 347   err |= audio_processing->gain_control()->Enable(true);
 348   CHECK_EQ(err, 0);
 349 }
 350
 351 void GetAecStats(webrtc::EchoCancellation* echo_cancellation,
 352                  webrtc::AudioProcessorInterface::AudioProcessorStats* stats) {
 353   // These values can take on valid negative values, so use the lowest possible
 354   // level as default rather than -1.
 355   stats->echo_return_loss = -100;
 356   stats->echo_return_loss_enhancement = -100;
 357
 358   // The median value can also be negative, but in practice -1 is only used to
 359   // signal insufficient data, since the resolution is limited to multiples
 360   // of 4ms.
 361   stats->echo_delay_median_ms = -1;
 362   stats->echo_delay_std_ms = -1;
 363
 364   // TODO(ajm): Re-enable this metric once we have a reliable implementation.
 365   stats->aec_quality_min = -1.0f;
 366
 367   if (!echo_cancellation->are_metrics_enabled() ||
 368       !echo_cancellation->is_delay_logging_enabled() ||
 369       !echo_cancellation->is_enabled()) {
 370     return;
 371   }
 372
 373   // TODO(ajm): we may want to use VoECallReport::GetEchoMetricsSummary
 374   // here, but it appears to be unsuitable currently. Revisit after this is
 375   // investigated: http://b/issue?id=5666755
 376   webrtc::EchoCancellation::Metrics echo_metrics;
 377   if (!echo_cancellation->GetMetrics(&echo_metrics)) {
 378     stats->echo_return_loss = echo_metrics.echo_return_loss.instant;
 379     stats->echo_return_loss_enhancement =
 380         echo_metrics.echo_return_loss_enhancement.instant;
 381   }
 382
 383   int median = 0, std = 0;
 384   float dummy = 0;
 385   if (echo_cancellation->GetDelayMetrics(&median, &std, &dummy) ==
 386       webrtc::AudioProcessing::kNoError) {
 387     stats->echo_delay_median_ms = median;
 388     stats->echo_delay_std_ms = std;
 389   }
 390 }
 391
 392 }  // namespace content