content/renderer/media/media_stream_audio_processor_options.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/media_stream_audio_processor_options.h"
   6
   7 #include "base/files/file_path.h"
   8 #include "base/files/file_util.h"
   9 #include "base/logging.h"
  10 #include "base/metrics/field_trial.h"
  11 #include "base/metrics/histogram.h"
  12 #include "base/strings/string_number_conversions.h"
  13 #include "base/strings/string_split.h"
  14 #include "base/strings/string_util.h"
  15 #include "base/strings/utf_string_conversions.h"
  16 #include "content/common/media/media_stream_options.h"
  17 #include "content/renderer/media/media_stream_constraints_util.h"
  18 #include "content/renderer/media/media_stream_source.h"
  19 #include "content/renderer/media/rtc_media_constraints.h"
  20 #include "media/audio/audio_parameters.h"
  21 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
  22 #include "third_party/webrtc/modules/audio_processing/typing_detection.h"
  23
  24 namespace content {
  25
  26 const char MediaAudioConstraints::kEchoCancellation[] = "echoCancellation";
  27 const char MediaAudioConstraints::kGoogEchoCancellation[] =
  28     "googEchoCancellation";
  29 const char MediaAudioConstraints::kGoogExperimentalEchoCancellation[] =
  30     "googEchoCancellation2";
  31 const char MediaAudioConstraints::kGoogAutoGainControl[] =
  32     "googAutoGainControl";
  33 const char MediaAudioConstraints::kGoogExperimentalAutoGainControl[] =
  34     "googAutoGainControl2";
  35 const char MediaAudioConstraints::kGoogNoiseSuppression[] =
  36     "googNoiseSuppression";
  37 const char MediaAudioConstraints::kGoogExperimentalNoiseSuppression[] =
  38     "googNoiseSuppression2";
  39 const char MediaAudioConstraints::kGoogBeamforming[] = "googBeamforming";
  40 const char MediaAudioConstraints::kGoogArrayGeometry[] = "googArrayGeometry";
  41 const char MediaAudioConstraints::kGoogHighpassFilter[] = "googHighpassFilter";
  42 const char MediaAudioConstraints::kGoogTypingNoiseDetection[] =
  43     "googTypingNoiseDetection";
  44 const char MediaAudioConstraints::kGoogAudioMirroring[] = "googAudioMirroring";
  45
  46 namespace {
  47
  48 // Constant constraint keys which enables default audio constraints on
  49 // mediastreams with audio.
  50 struct {
  51   const char* key;
  52   bool value;
  53 } const kDefaultAudioConstraints[] = {
  54   { MediaAudioConstraints::kEchoCancellation, true },
  55   { MediaAudioConstraints::kGoogEchoCancellation, true },
  56 #if defined(OS_ANDROID) || defined(OS_IOS)
  57   { MediaAudioConstraints::kGoogExperimentalEchoCancellation, false },
  58 #else
  59   // Enable the extended filter mode AEC on all non-mobile platforms.
  60   { MediaAudioConstraints::kGoogExperimentalEchoCancellation, true },
  61 #endif
  62   { MediaAudioConstraints::kGoogAutoGainControl, true },
  63   { MediaAudioConstraints::kGoogExperimentalAutoGainControl, true },
  64   { MediaAudioConstraints::kGoogNoiseSuppression, true },
  65   { MediaAudioConstraints::kGoogHighpassFilter, true },
  66   { MediaAudioConstraints::kGoogTypingNoiseDetection, true },
  67   { MediaAudioConstraints::kGoogExperimentalNoiseSuppression, false },
  68   // Beamforming will only be enabled if we are also provided with a
  69   // multi-microphone geometry.
  70   { MediaAudioConstraints::kGoogBeamforming, false },
  71   { kMediaStreamAudioHotword, false },
  72 };
  73
  74 // Used to log echo quality based on delay estimates.
  75 enum DelayBasedEchoQuality {
  76   DELAY_BASED_ECHO_QUALITY_GOOD = 0,
  77   DELAY_BASED_ECHO_QUALITY_SPURIOUS,
  78   DELAY_BASED_ECHO_QUALITY_BAD,
  79   DELAY_BASED_ECHO_QUALITY_INVALID,
  80   DELAY_BASED_ECHO_QUALITY_MAX
  81 };
  82
  83 DelayBasedEchoQuality EchoDelayFrequencyToQuality(float delay_frequency) {
  84   const float kEchoDelayFrequencyLowerLimit = 0.1f;
  85   const float kEchoDelayFrequencyUpperLimit = 0.8f;
  86   // DELAY_BASED_ECHO_QUALITY_GOOD
  87   //   delay is out of bounds during at most 10 % of the time.
  88   // DELAY_BASED_ECHO_QUALITY_SPURIOUS
  89   //   delay is out of bounds 10-80 % of the time.
  90   // DELAY_BASED_ECHO_QUALITY_BAD
  91   //   delay is mostly out of bounds >= 80 % of the time.
  92   // DELAY_BASED_ECHO_QUALITY_INVALID
  93   //   delay_frequency is negative which happens if we have insufficient data.
  94   if (delay_frequency < 0)
  95     return DELAY_BASED_ECHO_QUALITY_INVALID;
  96   else if (delay_frequency <= kEchoDelayFrequencyLowerLimit)
  97     return DELAY_BASED_ECHO_QUALITY_GOOD;
  98   else if (delay_frequency < kEchoDelayFrequencyUpperLimit)
  99     return DELAY_BASED_ECHO_QUALITY_SPURIOUS;
 100   else
 101     return DELAY_BASED_ECHO_QUALITY_BAD;
 102 }
 103
 104 webrtc::Point WebrtcPointFromMediaPoint(const media::Point& point) {
 105   return webrtc::Point(point.x(), point.y(), point.z());
 106 }
 107
 108 std::vector<webrtc::Point> WebrtcPointsFromMediaPoints(
 109     const std::vector<media::Point>& points) {
 110   std::vector<webrtc::Point> webrtc_points;
 111   webrtc_points.reserve(webrtc_points.size());
 112   for (const auto& point : points)
 113     webrtc_points.push_back(WebrtcPointFromMediaPoint(point));
 114   return webrtc_points;
 115 }
 116
 117 }  // namespace
 118
 119 // TODO(xians): Remove this method after the APM in WebRtc is deprecated.
 120 void MediaAudioConstraints::ApplyFixedAudioConstraints(
 121     RTCMediaConstraints* constraints) {
 122   for (size_t i = 0; i < arraysize(kDefaultAudioConstraints); ++i) {
 123     bool already_set_value;
 124     if (!webrtc::FindConstraint(constraints, kDefaultAudioConstraints[i].key,
 125                                 &already_set_value, NULL)) {
 126       const std::string value = kDefaultAudioConstraints[i].value ?
 127           webrtc::MediaConstraintsInterface::kValueTrue :
 128           webrtc::MediaConstraintsInterface::kValueFalse;
 129       constraints->AddOptional(kDefaultAudioConstraints[i].key, value, false);
 130     } else {
 131       DVLOG(1) << "Constraint " << kDefaultAudioConstraints[i].key
 132                << " already set to " << already_set_value;
 133     }
 134   }
 135 }
 136
 137 MediaAudioConstraints::MediaAudioConstraints(
 138     const blink::WebMediaConstraints& constraints, int effects)
 139     : constraints_(constraints),
 140       effects_(effects),
 141       default_audio_processing_constraint_value_(true) {
 142   // The default audio processing constraints are turned off when
 143   // - gUM has a specific kMediaStreamSource, which is used by tab capture
 144   //   and screen capture.
 145   // - |kEchoCancellation| is explicitly set to false.
 146   std::string value_str;
 147   bool value_bool = false;
 148   if ((GetConstraintValueAsString(constraints, kMediaStreamSource,
 149                                   &value_str)) ||
 150       (GetConstraintValueAsBoolean(constraints_, kEchoCancellation,
 151                                    &value_bool) && !value_bool)) {
 152     default_audio_processing_constraint_value_ = false;
 153   }
 154 }
 155
 156 MediaAudioConstraints::~MediaAudioConstraints() {}
 157
 158 bool MediaAudioConstraints::GetProperty(const std::string& key) const {
 159   // Return the value if the constraint is specified in |constraints|,
 160   // otherwise return the default value.
 161   bool value = false;
 162   if (!GetConstraintValueAsBoolean(constraints_, key, &value))
 163     value = GetDefaultValueForConstraint(constraints_, key);
 164
 165   return value;
 166 }
 167
 168 std::string MediaAudioConstraints::GetPropertyAsString(
 169     const std::string& key) const {
 170   std::string value;
 171   GetConstraintValueAsString(constraints_, key, &value);
 172   return value;
 173 }
 174
 175 bool MediaAudioConstraints::GetEchoCancellationProperty() const {
 176   // If platform echo canceller is enabled, disable the software AEC.
 177   if (effects_ & media::AudioParameters::ECHO_CANCELLER)
 178     return false;
 179
 180   // If |kEchoCancellation| is specified in the constraints, it will
 181   // override the value of |kGoogEchoCancellation|.
 182   bool value = false;
 183   if (GetConstraintValueAsBoolean(constraints_, kEchoCancellation, &value))
 184     return value;
 185
 186   return GetProperty(kGoogEchoCancellation);
 187 }
 188
 189 bool MediaAudioConstraints::IsValid() const {
 190   blink::WebVector<blink::WebMediaConstraint> mandatory;
 191   constraints_.getMandatoryConstraints(mandatory);
 192   for (size_t i = 0; i < mandatory.size(); ++i) {
 193     const std::string key = mandatory[i].m_name.utf8();
 194     if (key == kMediaStreamSource || key == kMediaStreamSourceId ||
 195         key == MediaStreamSource::kSourceId) {
 196       // Ignore Chrome specific Tab capture and |kSourceId| constraints.
 197       continue;
 198     }
 199
 200     bool valid = false;
 201     for (size_t j = 0; j < arraysize(kDefaultAudioConstraints); ++j) {
 202       if (key == kDefaultAudioConstraints[j].key) {
 203         bool value = false;
 204         valid = GetMandatoryConstraintValueAsBoolean(constraints_, key, &value);
 205         break;
 206       }
 207     }
 208
 209     if (!valid) {
 210       DLOG(ERROR) << "Invalid MediaStream constraint. Name: " << key;
 211       return false;
 212     }
 213   }
 214
 215   return true;
 216 }
 217
 218 bool MediaAudioConstraints::GetDefaultValueForConstraint(
 219     const blink::WebMediaConstraints& constraints,
 220     const std::string& key) const {
 221   if (!default_audio_processing_constraint_value_)
 222     return false;
 223
 224   for (size_t i = 0; i < arraysize(kDefaultAudioConstraints); ++i) {
 225     if (kDefaultAudioConstraints[i].key == key)
 226       return kDefaultAudioConstraints[i].value;
 227   }
 228
 229   return false;
 230 }
 231
 232 EchoInformation::EchoInformation()
 233     : num_chunks_(0), echo_frames_received_(false) {
 234 }
 235
 236 EchoInformation::~EchoInformation() {}
 237
 238 void EchoInformation::UpdateAecDelayStats(
 239     webrtc::EchoCancellation* echo_cancellation) {
 240   // Only start collecting stats if we know echo cancellation has measured an
 241   // echo. Otherwise we clutter the stats with for example cases where only the
 242   // microphone is used.
 243   if (!echo_frames_received_ & !echo_cancellation->stream_has_echo())
 244     return;
 245
 246   echo_frames_received_ = true;
 247   // In WebRTC, three echo delay metrics are calculated and updated every
 248   // five seconds. We use one of them, |fraction_poor_delays| to log in a UMA
 249   // histogram an Echo Cancellation quality metric. The stat in WebRTC has a
 250   // fixed aggregation window of five seconds, so we use the same query
 251   // frequency to avoid logging old values.
 252   const int kNumChunksInFiveSeconds = 500;
 253   if (!echo_cancellation->is_delay_logging_enabled() ||
 254       !echo_cancellation->is_enabled()) {
 255     return;
 256   }
 257
 258   num_chunks_++;
 259   if (num_chunks_ < kNumChunksInFiveSeconds) {
 260     return;
 261   }
 262
 263   int dummy_median = 0, dummy_std = 0;
 264   float fraction_poor_delays = 0;
 265   if (echo_cancellation->GetDelayMetrics(
 266           &dummy_median, &dummy_std, &fraction_poor_delays) ==
 267       webrtc::AudioProcessing::kNoError) {
 268     num_chunks_ = 0;
 269     // Map |fraction_poor_delays| to an Echo Cancellation quality and log in UMA
 270     // histogram. See DelayBasedEchoQuality for information on histogram
 271     // buckets.
 272     UMA_HISTOGRAM_ENUMERATION("WebRTC.AecDelayBasedQuality",
 273                               EchoDelayFrequencyToQuality(fraction_poor_delays),
 274                               DELAY_BASED_ECHO_QUALITY_MAX);
 275   }
 276 }
 277
 278 void EnableEchoCancellation(AudioProcessing* audio_processing) {
 279 #if defined(OS_ANDROID) || defined(OS_IOS)
 280   const std::string group_name =
 281       base::FieldTrialList::FindFullName("ReplaceAECMWithAEC");
 282   if (group_name.empty() ||
 283       !(group_name == "Enabled" || group_name == "DefaultEnabled")) {
 284     // Mobile devices are using AECM.
 285     int err = audio_processing->echo_control_mobile()->set_routing_mode(
 286         webrtc::EchoControlMobile::kSpeakerphone);
 287     err |= audio_processing->echo_control_mobile()->Enable(true);
 288     CHECK_EQ(err, 0);
 289     return;
 290   }
 291 #endif
 292   int err = audio_processing->echo_cancellation()->set_suppression_level(
 293       webrtc::EchoCancellation::kHighSuppression);
 294
 295   // Enable the metrics for AEC.
 296   err |= audio_processing->echo_cancellation()->enable_metrics(true);
 297   err |= audio_processing->echo_cancellation()->enable_delay_logging(true);
 298   err |= audio_processing->echo_cancellation()->Enable(true);
 299   CHECK_EQ(err, 0);
 300 }
 301
 302 void EnableNoiseSuppression(AudioProcessing* audio_processing,
 303                             webrtc::NoiseSuppression::Level ns_level) {
 304   int err = audio_processing->noise_suppression()->set_level(ns_level);
 305   err |= audio_processing->noise_suppression()->Enable(true);
 306   CHECK_EQ(err, 0);
 307 }
 308
 309 void EnableHighPassFilter(AudioProcessing* audio_processing) {
 310   CHECK_EQ(audio_processing->high_pass_filter()->Enable(true), 0);
 311 }
 312
 313 void EnableTypingDetection(AudioProcessing* audio_processing,
 314                            webrtc::TypingDetection* typing_detector) {
 315   int err = audio_processing->voice_detection()->Enable(true);
 316   err |= audio_processing->voice_detection()->set_likelihood(
 317       webrtc::VoiceDetection::kVeryLowLikelihood);
 318   CHECK_EQ(err, 0);
 319
 320   // Configure the update period to 1s (100 * 10ms) in the typing detector.
 321   typing_detector->SetParameters(0, 0, 0, 0, 0, 100);
 322 }
 323
 324 void StartEchoCancellationDump(AudioProcessing* audio_processing,
 325                                base::File aec_dump_file) {
 326   DCHECK(aec_dump_file.IsValid());
 327
 328   FILE* stream = base::FileToFILE(aec_dump_file.Pass(), "w");
 329   if (!stream) {
 330     LOG(ERROR) << "Failed to open AEC dump file";
 331     return;
 332   }
 333
 334   if (audio_processing->StartDebugRecording(stream))
 335     DLOG(ERROR) << "Fail to start AEC debug recording";
 336 }
 337
 338 void StopEchoCancellationDump(AudioProcessing* audio_processing) {
 339   if (audio_processing->StopDebugRecording())
 340     DLOG(ERROR) << "Fail to stop AEC debug recording";
 341 }
 342
 343 void EnableAutomaticGainControl(AudioProcessing* audio_processing) {
 344 #if defined(OS_ANDROID) || defined(OS_IOS)
 345   const webrtc::GainControl::Mode mode = webrtc::GainControl::kFixedDigital;
 346 #else
 347   const webrtc::GainControl::Mode mode = webrtc::GainControl::kAdaptiveAnalog;
 348 #endif
 349   int err = audio_processing->gain_control()->set_mode(mode);
 350   err |= audio_processing->gain_control()->Enable(true);
 351   CHECK_EQ(err, 0);
 352 }
 353
 354 void GetAecStats(webrtc::EchoCancellation* echo_cancellation,
 355                  webrtc::AudioProcessorInterface::AudioProcessorStats* stats) {
 356   // These values can take on valid negative values, so use the lowest possible
 357   // level as default rather than -1.
 358   stats->echo_return_loss = -100;
 359   stats->echo_return_loss_enhancement = -100;
 360
 361   // The median value can also be negative, but in practice -1 is only used to
 362   // signal insufficient data, since the resolution is limited to multiples
 363   // of 4ms.
 364   stats->echo_delay_median_ms = -1;
 365   stats->echo_delay_std_ms = -1;
 366
 367   // TODO(ajm): Re-enable this metric once we have a reliable implementation.
 368   stats->aec_quality_min = -1.0f;
 369
 370   if (!echo_cancellation->are_metrics_enabled() ||
 371       !echo_cancellation->is_delay_logging_enabled() ||
 372       !echo_cancellation->is_enabled()) {
 373     return;
 374   }
 375
 376   // TODO(ajm): we may want to use VoECallReport::GetEchoMetricsSummary
 377   // here, but it appears to be unsuitable currently. Revisit after this is
 378   // investigated: http://b/issue?id=5666755
 379   webrtc::EchoCancellation::Metrics echo_metrics;
 380   if (!echo_cancellation->GetMetrics(&echo_metrics)) {
 381     stats->echo_return_loss = echo_metrics.echo_return_loss.instant;
 382     stats->echo_return_loss_enhancement =
 383         echo_metrics.echo_return_loss_enhancement.instant;
 384   }
 385
 386   int median = 0, std = 0;
 387   float dummy = 0;
 388   if (echo_cancellation->GetDelayMetrics(&median, &std, &dummy) ==
 389       webrtc::AudioProcessing::kNoError) {
 390     stats->echo_delay_median_ms = median;
 391     stats->echo_delay_std_ms = std;
 392   }
 393 }
 394
 395 std::vector<webrtc::Point> GetArrayGeometryPreferringConstraints(
 396     const MediaAudioConstraints& audio_constraints,
 397     const MediaStreamDevice::AudioDeviceParameters& input_params) {
 398   const std::string constraints_geometry =
 399       audio_constraints.GetPropertyAsString(
 400           MediaAudioConstraints::kGoogArrayGeometry);
 401
 402   // Give preference to the audio constraint over the device-supplied mic
 403   // positions. This is mainly for testing purposes.
 404   return WebrtcPointsFromMediaPoints(
 405       constraints_geometry.empty()
 406           ? input_params.mic_positions
 407           : media::ParsePointsFromString(constraints_geometry));
 408 }
 409
 410 }  // namespace content