Re-subimission of https://codereview.chromium.org/1041213003/
[chromium-blink-merge.git] / content / renderer / media / media_stream_audio_processor_options.cc
blobc39b1544c5c878a13abf2a7c4b64a41319d978f1
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/media_stream_audio_processor_options.h"
7 #include "base/files/file_path.h"
8 #include "base/files/file_util.h"
9 #include "base/logging.h"
10 #include "base/metrics/field_trial.h"
11 #include "base/metrics/histogram.h"
12 #include "base/strings/string_number_conversions.h"
13 #include "base/strings/utf_string_conversions.h"
14 #include "content/common/media/media_stream_options.h"
15 #include "content/renderer/media/media_stream_constraints_util.h"
16 #include "content/renderer/media/media_stream_source.h"
17 #include "content/renderer/media/rtc_media_constraints.h"
18 #include "media/audio/audio_parameters.h"
19 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
20 #include "third_party/webrtc/modules/audio_processing/typing_detection.h"
22 namespace content {
24 const char MediaAudioConstraints::kEchoCancellation[] = "echoCancellation";
25 const char MediaAudioConstraints::kGoogEchoCancellation[] =
26 "googEchoCancellation";
27 const char MediaAudioConstraints::kGoogExperimentalEchoCancellation[] =
28 "googEchoCancellation2";
29 const char MediaAudioConstraints::kGoogAutoGainControl[] =
30 "googAutoGainControl";
31 const char MediaAudioConstraints::kGoogExperimentalAutoGainControl[] =
32 "googAutoGainControl2";
33 const char MediaAudioConstraints::kGoogNoiseSuppression[] =
34 "googNoiseSuppression";
35 const char MediaAudioConstraints::kGoogExperimentalNoiseSuppression[] =
36 "googNoiseSuppression2";
37 const char MediaAudioConstraints::kGoogBeamforming[] = "googBeamforming";
38 const char MediaAudioConstraints::kGoogHighpassFilter[] = "googHighpassFilter";
39 const char MediaAudioConstraints::kGoogTypingNoiseDetection[] =
40 "googTypingNoiseDetection";
41 const char MediaAudioConstraints::kGoogAudioMirroring[] = "googAudioMirroring";
42 const char MediaAudioConstraints::kGoogAudioProcessing48kHzSupport[] =
43 "googAudioProcessing48kHzSupport";
45 namespace {
47 // Constant constraint keys which enables default audio constraints on
48 // mediastreams with audio.
49 struct {
50 const char* key;
51 bool value;
52 } const kDefaultAudioConstraints[] = {
53 { MediaAudioConstraints::kEchoCancellation, true },
54 { MediaAudioConstraints::kGoogEchoCancellation, true },
55 #if defined(OS_ANDROID) || defined(OS_IOS)
56 { MediaAudioConstraints::kGoogExperimentalEchoCancellation, false },
57 #else
58 // Enable the extended filter mode AEC on all non-mobile platforms.
59 { MediaAudioConstraints::kGoogExperimentalEchoCancellation, true },
60 #endif
61 { MediaAudioConstraints::kGoogAutoGainControl, true },
62 { MediaAudioConstraints::kGoogExperimentalAutoGainControl, true },
63 { MediaAudioConstraints::kGoogNoiseSuppression, true },
64 { MediaAudioConstraints::kGoogHighpassFilter, true },
65 { MediaAudioConstraints::kGoogTypingNoiseDetection, true },
66 { MediaAudioConstraints::kGoogExperimentalNoiseSuppression, false },
67 { MediaAudioConstraints::kGoogBeamforming, false },
68 #if defined(OS_WIN)
69 { kMediaStreamAudioDucking, true },
70 #else
71 { kMediaStreamAudioDucking, false },
72 #endif
73 { kMediaStreamAudioHotword, false },
74 { MediaAudioConstraints::kGoogAudioProcessing48kHzSupport, false },
77 bool IsAudioProcessingConstraint(const std::string& key) {
78 // |kMediaStreamAudioDucking| does not require audio processing.
79 return key != kMediaStreamAudioDucking;
82 // Used to log echo quality based on delay estimates.
83 enum DelayBasedEchoQuality {
84 DELAY_BASED_ECHO_QUALITY_GOOD = 0,
85 DELAY_BASED_ECHO_QUALITY_SPURIOUS,
86 DELAY_BASED_ECHO_QUALITY_BAD,
87 DELAY_BASED_ECHO_QUALITY_INVALID,
88 DELAY_BASED_ECHO_QUALITY_MAX
91 DelayBasedEchoQuality EchoDelayFrequencyToQuality(float delay_frequency) {
92 const float kEchoDelayFrequencyLowerLimit = 0.1f;
93 const float kEchoDelayFrequencyUpperLimit = 0.8f;
94 // DELAY_BASED_ECHO_QUALITY_GOOD
95 // delay is out of bounds during at most 10 % of the time.
96 // DELAY_BASED_ECHO_QUALITY_SPURIOUS
97 // delay is out of bounds 10-80 % of the time.
98 // DELAY_BASED_ECHO_QUALITY_BAD
99 // delay is mostly out of bounds >= 80 % of the time.
100 // DELAY_BASED_ECHO_QUALITY_INVALID
101 // delay_frequency is negative which happens if we have insufficient data.
102 if (delay_frequency < 0)
103 return DELAY_BASED_ECHO_QUALITY_INVALID;
104 else if (delay_frequency <= kEchoDelayFrequencyLowerLimit)
105 return DELAY_BASED_ECHO_QUALITY_GOOD;
106 else if (delay_frequency < kEchoDelayFrequencyUpperLimit)
107 return DELAY_BASED_ECHO_QUALITY_SPURIOUS;
108 else
109 return DELAY_BASED_ECHO_QUALITY_BAD;
112 } // namespace
114 // TODO(xians): Remove this method after the APM in WebRtc is deprecated.
115 void MediaAudioConstraints::ApplyFixedAudioConstraints(
116 RTCMediaConstraints* constraints) {
117 for (size_t i = 0; i < arraysize(kDefaultAudioConstraints); ++i) {
118 bool already_set_value;
119 if (!webrtc::FindConstraint(constraints, kDefaultAudioConstraints[i].key,
120 &already_set_value, NULL)) {
121 const std::string value = kDefaultAudioConstraints[i].value ?
122 webrtc::MediaConstraintsInterface::kValueTrue :
123 webrtc::MediaConstraintsInterface::kValueFalse;
124 constraints->AddOptional(kDefaultAudioConstraints[i].key, value, false);
125 } else {
126 DVLOG(1) << "Constraint " << kDefaultAudioConstraints[i].key
127 << " already set to " << already_set_value;
132 MediaAudioConstraints::MediaAudioConstraints(
133 const blink::WebMediaConstraints& constraints, int effects)
134 : constraints_(constraints),
135 effects_(effects),
136 default_audio_processing_constraint_value_(true) {
137 // The default audio processing constraints are turned off when
138 // - gUM has a specific kMediaStreamSource, which is used by tab capture
139 // and screen capture.
140 // - |kEchoCancellation| is explicitly set to false.
141 std::string value_str;
142 bool value_bool = false;
143 if ((GetConstraintValueAsString(constraints, kMediaStreamSource,
144 &value_str)) ||
145 (GetConstraintValueAsBoolean(constraints_, kEchoCancellation,
146 &value_bool) && !value_bool)) {
147 default_audio_processing_constraint_value_ = false;
151 MediaAudioConstraints::~MediaAudioConstraints() {}
153 bool MediaAudioConstraints::GetProperty(const std::string& key) const {
154 // Return the value if the constraint is specified in |constraints|,
155 // otherwise return the default value.
156 bool value = false;
157 if (!GetConstraintValueAsBoolean(constraints_, key, &value))
158 value = GetDefaultValueForConstraint(constraints_, key);
160 return value;
163 bool MediaAudioConstraints::GetEchoCancellationProperty() const {
164 // If platform echo canceller is enabled, disable the software AEC.
165 if (effects_ & media::AudioParameters::ECHO_CANCELLER)
166 return false;
168 // If |kEchoCancellation| is specified in the constraints, it will
169 // override the value of |kGoogEchoCancellation|.
170 bool value = false;
171 if (GetConstraintValueAsBoolean(constraints_, kEchoCancellation, &value))
172 return value;
174 return GetProperty(kGoogEchoCancellation);
177 bool MediaAudioConstraints::IsValid() const {
178 blink::WebVector<blink::WebMediaConstraint> mandatory;
179 constraints_.getMandatoryConstraints(mandatory);
180 for (size_t i = 0; i < mandatory.size(); ++i) {
181 const std::string key = mandatory[i].m_name.utf8();
182 if (key == kMediaStreamSource || key == kMediaStreamSourceId ||
183 key == MediaStreamSource::kSourceId) {
184 // Ignore Chrome specific Tab capture and |kSourceId| constraints.
185 continue;
188 bool valid = false;
189 for (size_t j = 0; j < arraysize(kDefaultAudioConstraints); ++j) {
190 if (key == kDefaultAudioConstraints[j].key) {
191 bool value = false;
192 valid = GetMandatoryConstraintValueAsBoolean(constraints_, key, &value);
193 break;
197 if (!valid) {
198 DLOG(ERROR) << "Invalid MediaStream constraint. Name: " << key;
199 return false;
203 return true;
206 bool MediaAudioConstraints::GetDefaultValueForConstraint(
207 const blink::WebMediaConstraints& constraints,
208 const std::string& key) const {
209 // |kMediaStreamAudioDucking| is not restricted by
210 // |default_audio_processing_constraint_value_| since it does not require
211 // audio processing.
212 if (!default_audio_processing_constraint_value_ &&
213 IsAudioProcessingConstraint(key))
214 return false;
216 for (size_t i = 0; i < arraysize(kDefaultAudioConstraints); ++i) {
217 if (kDefaultAudioConstraints[i].key == key)
218 return kDefaultAudioConstraints[i].value;
221 return false;
224 EchoInformation::EchoInformation()
225 : num_chunks_(0) {}
227 EchoInformation::~EchoInformation() {}
229 void EchoInformation::UpdateAecDelayStats(
230 webrtc::EchoCancellation* echo_cancellation) {
231 // In WebRTC, three echo delay metrics are calculated and updated every
232 // five seconds. We use one of them, |fraction_poor_delays| to log in a UMA
233 // histogram an Echo Cancellation quality metric. The stat in WebRTC has a
234 // fixed aggregation window of five seconds, so we use the same query
235 // frequency to avoid logging old values.
236 const int kNumChunksInFiveSeconds = 500;
237 if (!echo_cancellation->is_delay_logging_enabled() ||
238 !echo_cancellation->is_enabled()) {
239 return;
242 num_chunks_++;
243 if (num_chunks_ < kNumChunksInFiveSeconds) {
244 return;
247 int dummy_median = 0, dummy_std = 0;
248 float fraction_poor_delays = 0;
249 if (echo_cancellation->GetDelayMetrics(
250 &dummy_median, &dummy_std, &fraction_poor_delays) ==
251 webrtc::AudioProcessing::kNoError) {
252 num_chunks_ = 0;
253 // Map |fraction_poor_delays| to an Echo Cancellation quality and log in UMA
254 // histogram. See DelayBasedEchoQuality for information on histogram
255 // buckets.
256 UMA_HISTOGRAM_ENUMERATION("WebRTC.AecDelayBasedQuality",
257 EchoDelayFrequencyToQuality(fraction_poor_delays),
258 DELAY_BASED_ECHO_QUALITY_MAX);
262 void EnableEchoCancellation(AudioProcessing* audio_processing) {
263 #if defined(OS_ANDROID) || defined(OS_IOS)
264 const std::string group_name =
265 base::FieldTrialList::FindFullName("ReplaceAECMWithAEC");
266 if (group_name.empty() ||
267 !(group_name == "Enabled" || group_name == "DefaultEnabled")) {
268 // Mobile devices are using AECM.
269 int err = audio_processing->echo_control_mobile()->set_routing_mode(
270 webrtc::EchoControlMobile::kSpeakerphone);
271 err |= audio_processing->echo_control_mobile()->Enable(true);
272 CHECK_EQ(err, 0);
273 return;
275 #endif
276 int err = audio_processing->echo_cancellation()->set_suppression_level(
277 webrtc::EchoCancellation::kHighSuppression);
279 // Enable the metrics for AEC.
280 err |= audio_processing->echo_cancellation()->enable_metrics(true);
281 err |= audio_processing->echo_cancellation()->enable_delay_logging(true);
282 err |= audio_processing->echo_cancellation()->Enable(true);
283 CHECK_EQ(err, 0);
286 void EnableNoiseSuppression(AudioProcessing* audio_processing) {
287 int err = audio_processing->noise_suppression()->set_level(
288 webrtc::NoiseSuppression::kHigh);
289 err |= audio_processing->noise_suppression()->Enable(true);
290 CHECK_EQ(err, 0);
293 void EnableHighPassFilter(AudioProcessing* audio_processing) {
294 CHECK_EQ(audio_processing->high_pass_filter()->Enable(true), 0);
297 void EnableTypingDetection(AudioProcessing* audio_processing,
298 webrtc::TypingDetection* typing_detector) {
299 int err = audio_processing->voice_detection()->Enable(true);
300 err |= audio_processing->voice_detection()->set_likelihood(
301 webrtc::VoiceDetection::kVeryLowLikelihood);
302 CHECK_EQ(err, 0);
304 // Configure the update period to 1s (100 * 10ms) in the typing detector.
305 typing_detector->SetParameters(0, 0, 0, 0, 0, 100);
308 void StartEchoCancellationDump(AudioProcessing* audio_processing,
309 base::File aec_dump_file) {
310 DCHECK(aec_dump_file.IsValid());
312 FILE* stream = base::FileToFILE(aec_dump_file.Pass(), "w");
313 if (!stream) {
314 LOG(ERROR) << "Failed to open AEC dump file";
315 return;
318 if (audio_processing->StartDebugRecording(stream))
319 DLOG(ERROR) << "Fail to start AEC debug recording";
322 void StopEchoCancellationDump(AudioProcessing* audio_processing) {
323 if (audio_processing->StopDebugRecording())
324 DLOG(ERROR) << "Fail to stop AEC debug recording";
327 void EnableAutomaticGainControl(AudioProcessing* audio_processing) {
328 #if defined(OS_ANDROID) || defined(OS_IOS)
329 const webrtc::GainControl::Mode mode = webrtc::GainControl::kFixedDigital;
330 #else
331 const webrtc::GainControl::Mode mode = webrtc::GainControl::kAdaptiveAnalog;
332 #endif
333 int err = audio_processing->gain_control()->set_mode(mode);
334 err |= audio_processing->gain_control()->Enable(true);
335 CHECK_EQ(err, 0);
338 void GetAecStats(webrtc::EchoCancellation* echo_cancellation,
339 webrtc::AudioProcessorInterface::AudioProcessorStats* stats) {
340 // These values can take on valid negative values, so use the lowest possible
341 // level as default rather than -1.
342 stats->echo_return_loss = -100;
343 stats->echo_return_loss_enhancement = -100;
345 // The median value can also be negative, but in practice -1 is only used to
346 // signal insufficient data, since the resolution is limited to multiples
347 // of 4ms.
348 stats->echo_delay_median_ms = -1;
349 stats->echo_delay_std_ms = -1;
351 // TODO(ajm): Re-enable this metric once we have a reliable implementation.
352 stats->aec_quality_min = -1.0f;
354 if (!echo_cancellation->are_metrics_enabled() ||
355 !echo_cancellation->is_delay_logging_enabled() ||
356 !echo_cancellation->is_enabled()) {
357 return;
360 // TODO(ajm): we may want to use VoECallReport::GetEchoMetricsSummary
361 // here, but it appears to be unsuitable currently. Revisit after this is
362 // investigated: http://b/issue?id=5666755
363 webrtc::EchoCancellation::Metrics echo_metrics;
364 if (!echo_cancellation->GetMetrics(&echo_metrics)) {
365 stats->echo_return_loss = echo_metrics.echo_return_loss.instant;
366 stats->echo_return_loss_enhancement =
367 echo_metrics.echo_return_loss_enhancement.instant;
370 int median = 0, std = 0;
371 float dummy = 0;
372 if (echo_cancellation->GetDelayMetrics(&median, &std, &dummy) ==
373 webrtc::AudioProcessing::kNoError) {
374 stats->echo_delay_median_ms = median;
375 stats->echo_delay_std_ms = std;
379 } // namespace content