1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "media/audio/audio_input_controller.h"
8 #include "base/strings/string_number_conversions.h"
9 #include "base/strings/stringprintf.h"
10 #include "base/threading/thread_restrictions.h"
11 #include "base/time/time.h"
12 #include "media/audio/audio_parameters.h"
13 #include "media/base/scoped_histogram_timer.h"
14 #include "media/base/user_input_monitor.h"
16 using base::TimeDelta
;
20 const int kMaxInputChannels
= 3;
22 // TODO(henrika): remove usage of timers and add support for proper
23 // notification of when the input device is removed. This was originally added
24 // to resolve http://crbug.com/79936 for Windows platforms. This then caused
25 // breakage (very hard to repro bugs!) on other platforms: See
26 // http://crbug.com/226327 and http://crbug.com/230972.
27 // See also that the timer has been disabled on Mac now due to
29 const int kTimerResetIntervalSeconds
= 1;
30 // We have received reports that the timer can be too trigger happy on some
31 // Mac devices and the initial timer interval has therefore been increased
32 // from 1 second to 5 seconds.
33 const int kTimerInitialIntervalSeconds
= 5;
35 #if defined(AUDIO_POWER_MONITORING)
36 // Time in seconds between two successive measurements of audio power levels.
37 const int kPowerMonitorLogIntervalSeconds
= 15;
39 // A warning will be logged when the microphone audio volume is below this
41 const int kLowLevelMicrophoneLevelPercent
= 10;
43 // Logs if the user has enabled the microphone mute or not. This is normally
44 // done by marking a checkbox in an audio-settings UI which is unique for each
45 // platform. Elements in this enum should not be added, deleted or rearranged.
46 enum MicrophoneMuteResult
{
47 MICROPHONE_IS_MUTED
= 0,
48 MICROPHONE_IS_NOT_MUTED
= 1,
49 MICROPHONE_MUTE_MAX
= MICROPHONE_IS_NOT_MUTED
52 void LogMicrophoneMuteResult(MicrophoneMuteResult result
) {
53 UMA_HISTOGRAM_ENUMERATION("Media.MicrophoneMuted",
55 MICROPHONE_MUTE_MAX
+ 1);
58 // Helper method which calculates the average power of an audio bus. Unit is in
59 // dBFS, where 0 dBFS corresponds to all channels and samples equal to 1.0.
60 float AveragePower(const media::AudioBus
& buffer
) {
61 const int frames
= buffer
.frames();
62 const int channels
= buffer
.channels();
63 if (frames
<= 0 || channels
<= 0)
66 // Scan all channels and accumulate the sum of squares for all samples.
67 float sum_power
= 0.0f
;
68 for (int ch
= 0; ch
< channels
; ++ch
) {
69 const float* channel_data
= buffer
.channel(ch
);
70 for (int i
= 0; i
< frames
; i
++) {
71 const float sample
= channel_data
[i
];
72 sum_power
+= sample
* sample
;
76 // Update accumulated average results, with clamping for sanity.
77 const float average_power
=
78 std::max(0.0f
, std::min(1.0f
, sum_power
/ (frames
* channels
)));
80 // Convert average power level to dBFS units, and pin it down to zero if it
81 // is insignificantly small.
82 const float kInsignificantPower
= 1.0e-10f
; // -100 dBFS
83 const float power_dbfs
= average_power
< kInsignificantPower
?
84 -std::numeric_limits
<float>::infinity() : 10.0f
* log10f(average_power
);
88 #endif // AUDIO_POWER_MONITORING
92 // Used to log the result of capture startup.
93 // This was previously logged as a boolean with only the no callback and OK
94 // options. The enum order is kept to ensure backwards compatibility.
95 // Elements in this enum should not be deleted or rearranged; the only
96 // permitted operation is to add new elements before CAPTURE_STARTUP_RESULT_MAX
97 // and update CAPTURE_STARTUP_RESULT_MAX.
98 enum CaptureStartupResult
{
99 CAPTURE_STARTUP_NO_DATA_CALLBACK
= 0,
100 CAPTURE_STARTUP_OK
= 1,
101 CAPTURE_STARTUP_CREATE_STREAM_FAILED
= 2,
102 CAPTURE_STARTUP_OPEN_STREAM_FAILED
= 3,
103 CAPTURE_STARTUP_RESULT_MAX
= CAPTURE_STARTUP_OPEN_STREAM_FAILED
106 void LogCaptureStartupResult(CaptureStartupResult result
) {
107 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerCaptureStartupSuccess",
109 CAPTURE_STARTUP_RESULT_MAX
+ 1);
116 AudioInputController::Factory
* AudioInputController::factory_
= NULL
;
118 AudioInputController::AudioInputController(EventHandler
* handler
,
119 SyncWriter
* sync_writer
,
120 UserInputMonitor
* user_input_monitor
,
121 const bool agc_is_enabled
)
122 : creator_task_runner_(base::MessageLoopProxy::current()),
125 data_is_active_(false),
127 sync_writer_(sync_writer
),
129 user_input_monitor_(user_input_monitor
),
130 agc_is_enabled_(agc_is_enabled
),
131 #if defined(AUDIO_POWER_MONITORING)
132 power_measurement_is_enabled_(false),
133 log_silence_state_(false),
134 silence_state_(SILENCE_STATE_NO_MEASUREMENT
),
136 prev_key_down_count_(0) {
137 DCHECK(creator_task_runner_
.get());
140 AudioInputController::~AudioInputController() {
141 DCHECK_EQ(state_
, CLOSED
);
145 scoped_refptr
<AudioInputController
> AudioInputController::Create(
146 AudioManager
* audio_manager
,
147 EventHandler
* event_handler
,
148 const AudioParameters
& params
,
149 const std::string
& device_id
,
150 UserInputMonitor
* user_input_monitor
) {
151 DCHECK(audio_manager
);
153 if (!params
.IsValid() || (params
.channels() > kMaxInputChannels
))
157 return factory_
->Create(
158 audio_manager
, event_handler
, params
, user_input_monitor
);
160 scoped_refptr
<AudioInputController
> controller(
161 new AudioInputController(event_handler
, NULL
, user_input_monitor
, false));
163 controller
->task_runner_
= audio_manager
->GetTaskRunner();
165 // Create and open a new audio input stream from the existing
166 // audio-device thread.
167 if (!controller
->task_runner_
->PostTask(
169 base::Bind(&AudioInputController::DoCreate
,
171 base::Unretained(audio_manager
),
181 scoped_refptr
<AudioInputController
> AudioInputController::CreateLowLatency(
182 AudioManager
* audio_manager
,
183 EventHandler
* event_handler
,
184 const AudioParameters
& params
,
185 const std::string
& device_id
,
186 SyncWriter
* sync_writer
,
187 UserInputMonitor
* user_input_monitor
,
188 const bool agc_is_enabled
) {
189 DCHECK(audio_manager
);
192 if (!params
.IsValid() || (params
.channels() > kMaxInputChannels
))
195 // Create the AudioInputController object and ensure that it runs on
196 // the audio-manager thread.
197 scoped_refptr
<AudioInputController
> controller(new AudioInputController(
198 event_handler
, sync_writer
, user_input_monitor
, agc_is_enabled
));
199 controller
->task_runner_
= audio_manager
->GetTaskRunner();
201 // Create and open a new audio input stream from the existing
202 // audio-device thread. Use the provided audio-input device.
203 if (!controller
->task_runner_
->PostTask(
205 base::Bind(&AudioInputController::DoCreateForLowLatency
,
207 base::Unretained(audio_manager
),
217 scoped_refptr
<AudioInputController
> AudioInputController::CreateForStream(
218 const scoped_refptr
<base::SingleThreadTaskRunner
>& task_runner
,
219 EventHandler
* event_handler
,
220 AudioInputStream
* stream
,
221 SyncWriter
* sync_writer
,
222 UserInputMonitor
* user_input_monitor
) {
226 // Create the AudioInputController object and ensure that it runs on
227 // the audio-manager thread.
228 scoped_refptr
<AudioInputController
> controller(new AudioInputController(
229 event_handler
, sync_writer
, user_input_monitor
, false));
230 controller
->task_runner_
= task_runner
;
232 // TODO(miu): See TODO at top of file. Until that's resolved, we need to
233 // disable the error auto-detection here (since the audio mirroring
234 // implementation will reliably report error and close events). Note, of
235 // course, that we're assuming CreateForStream() has been called for the audio
236 // mirroring use case only.
237 if (!controller
->task_runner_
->PostTask(
239 base::Bind(&AudioInputController::DoCreateForStream
,
248 void AudioInputController::Record() {
249 task_runner_
->PostTask(FROM_HERE
, base::Bind(
250 &AudioInputController::DoRecord
, this));
253 void AudioInputController::Close(const base::Closure
& closed_task
) {
254 DCHECK(!closed_task
.is_null());
255 DCHECK(creator_task_runner_
->BelongsToCurrentThread());
257 task_runner_
->PostTaskAndReply(
258 FROM_HERE
, base::Bind(&AudioInputController::DoClose
, this), closed_task
);
261 void AudioInputController::SetVolume(double volume
) {
262 task_runner_
->PostTask(FROM_HERE
, base::Bind(
263 &AudioInputController::DoSetVolume
, this, volume
));
266 void AudioInputController::DoCreate(AudioManager
* audio_manager
,
267 const AudioParameters
& params
,
268 const std::string
& device_id
) {
269 DCHECK(task_runner_
->BelongsToCurrentThread());
270 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CreateTime");
272 handler_
->OnLog(this, "AIC::DoCreate");
274 #if defined(AUDIO_POWER_MONITORING)
275 // Disable power monitoring for streams that run without AGC enabled to
276 // avoid adding logs and UMA for non-WebRTC clients.
277 power_measurement_is_enabled_
= agc_is_enabled_
;
278 last_audio_level_log_time_
= base::TimeTicks::Now();
279 silence_state_
= SILENCE_STATE_NO_MEASUREMENT
;
282 // TODO(miu): See TODO at top of file. Until that's resolved, assume all
283 // platform audio input requires the |no_data_timer_| be used to auto-detect
284 // errors. In reality, probably only Windows needs to be treated as
286 DoCreateForStream(audio_manager
->MakeAudioInputStream(params
, device_id
));
289 void AudioInputController::DoCreateForLowLatency(AudioManager
* audio_manager
,
290 const AudioParameters
& params
,
291 const std::string
& device_id
) {
292 DCHECK(task_runner_
->BelongsToCurrentThread());
294 #if defined(AUDIO_POWER_MONITORING)
295 // We only log silence state UMA stats for low latency mode and if we use a
297 if (params
.format() != AudioParameters::AUDIO_FAKE
)
298 log_silence_state_
= true;
301 low_latency_create_time_
= base::TimeTicks::Now();
302 DoCreate(audio_manager
, params
, device_id
);
305 void AudioInputController::DoCreateForStream(
306 AudioInputStream
* stream_to_control
) {
307 DCHECK(task_runner_
->BelongsToCurrentThread());
310 stream_
= stream_to_control
;
314 handler_
->OnError(this, STREAM_CREATE_ERROR
);
315 LogCaptureStartupResult(CAPTURE_STARTUP_CREATE_STREAM_FAILED
);
319 if (stream_
&& !stream_
->Open()) {
323 handler_
->OnError(this, STREAM_OPEN_ERROR
);
324 LogCaptureStartupResult(CAPTURE_STARTUP_OPEN_STREAM_FAILED
);
328 DCHECK(!no_data_timer_
.get());
330 // Set AGC state using mode in |agc_is_enabled_| which can only be enabled in
331 // CreateLowLatency().
332 stream_
->SetAutomaticGainControl(agc_is_enabled_
);
334 // Create the data timer which will call FirstCheckForNoData(). The timer
335 // is started in DoRecord() and restarted in each DoCheckForNoData()
337 // The timer is enabled for logging purposes. The NO_DATA_ERROR triggered
338 // from the timer must be ignored by the EventHandler.
339 // TODO(henrika): remove usage of timer when it has been verified on Canary
340 // that we are safe doing so. Goal is to get rid of |no_data_timer_| and
341 // everything that is tied to it. crbug.com/357569.
342 no_data_timer_
.reset(new base::Timer(
343 FROM_HERE
, base::TimeDelta::FromSeconds(kTimerInitialIntervalSeconds
),
344 base::Bind(&AudioInputController::FirstCheckForNoData
,
345 base::Unretained(this)), false));
349 handler_
->OnCreated(this);
351 if (user_input_monitor_
) {
352 user_input_monitor_
->EnableKeyPressMonitoring();
353 prev_key_down_count_
= user_input_monitor_
->GetKeyPressCount();
357 void AudioInputController::DoRecord() {
358 DCHECK(task_runner_
->BelongsToCurrentThread());
359 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.RecordTime");
361 if (state_
!= CREATED
)
365 base::AutoLock
auto_lock(lock_
);
370 handler_
->OnLog(this, "AIC::DoRecord");
372 if (no_data_timer_
) {
373 // Start the data timer. Once |kTimerResetIntervalSeconds| have passed,
374 // a callback to FirstCheckForNoData() is made.
375 no_data_timer_
->Reset();
378 stream_
->Start(this);
380 handler_
->OnRecording(this);
383 void AudioInputController::DoClose() {
384 DCHECK(task_runner_
->BelongsToCurrentThread());
385 SCOPED_UMA_HISTOGRAM_TIMER("Media.AudioInputController.CloseTime");
387 if (state_
== CLOSED
)
390 // If this is a low-latency stream, log the total duration (since DoCreate)
391 // and add it to a UMA histogram.
392 if (!low_latency_create_time_
.is_null()) {
393 base::TimeDelta duration
=
394 base::TimeTicks::Now() - low_latency_create_time_
;
395 UMA_HISTOGRAM_LONG_TIMES("Media.InputStreamDuration", duration
);
397 std::string log_string
=
398 base::StringPrintf("AIC::DoClose: stream duration=");
399 log_string
+= base::Int64ToString(duration
.InSeconds());
400 log_string
+= " seconds";
401 handler_
->OnLog(this, log_string
);
405 // Delete the timer on the same thread that created it.
406 no_data_timer_
.reset();
408 DoStopCloseAndClearStream();
409 SetDataIsActive(false);
411 if (SharedMemoryAndSyncSocketMode())
412 sync_writer_
->Close();
414 if (user_input_monitor_
)
415 user_input_monitor_
->DisableKeyPressMonitoring();
417 #if defined(AUDIO_POWER_MONITORING)
418 // Send UMA stats if enabled.
419 if (log_silence_state_
)
420 LogSilenceState(silence_state_
);
421 log_silence_state_
= false;
427 void AudioInputController::DoReportError() {
428 DCHECK(task_runner_
->BelongsToCurrentThread());
430 handler_
->OnError(this, STREAM_ERROR
);
433 void AudioInputController::DoSetVolume(double volume
) {
434 DCHECK(task_runner_
->BelongsToCurrentThread());
435 DCHECK_GE(volume
, 0);
436 DCHECK_LE(volume
, 1.0);
438 if (state_
!= CREATED
&& state_
!= RECORDING
)
441 // Only ask for the maximum volume at first call and use cached value
442 // for remaining function calls.
444 max_volume_
= stream_
->GetMaxVolume();
447 if (max_volume_
== 0.0) {
448 DLOG(WARNING
) << "Failed to access input volume control";
452 // Set the stream volume and scale to a range matched to the platform.
453 stream_
->SetVolume(max_volume_
* volume
);
456 void AudioInputController::FirstCheckForNoData() {
457 DCHECK(task_runner_
->BelongsToCurrentThread());
458 LogCaptureStartupResult(GetDataIsActive() ?
460 CAPTURE_STARTUP_NO_DATA_CALLBACK
);
462 handler_
->OnLog(this, GetDataIsActive() ?
463 "AIC::FirstCheckForNoData => data is active" :
464 "AIC::FirstCheckForNoData => data is NOT active");
469 void AudioInputController::DoCheckForNoData() {
470 DCHECK(task_runner_
->BelongsToCurrentThread());
472 if (!GetDataIsActive()) {
473 // The data-is-active marker will be false only if it has been more than
474 // one second since a data packet was recorded. This can happen if a
475 // capture device has been removed or disabled.
477 handler_
->OnError(this, NO_DATA_ERROR
);
480 // Mark data as non-active. The flag will be re-enabled in OnData() each
481 // time a data packet is received. Hence, under normal conditions, the
482 // flag will only be disabled during a very short period.
483 SetDataIsActive(false);
485 // Restart the timer to ensure that we check the flag again in
486 // |kTimerResetIntervalSeconds|.
487 no_data_timer_
->Start(
488 FROM_HERE
, base::TimeDelta::FromSeconds(kTimerResetIntervalSeconds
),
489 base::Bind(&AudioInputController::DoCheckForNoData
,
490 base::Unretained(this)));
493 void AudioInputController::OnData(AudioInputStream
* stream
,
494 const AudioBus
* source
,
495 uint32 hardware_delay_bytes
,
497 // Mark data as active to ensure that the periodic calls to
498 // DoCheckForNoData() does not report an error to the event handler.
499 SetDataIsActive(true);
502 base::AutoLock
auto_lock(lock_
);
503 if (state_
!= RECORDING
)
507 bool key_pressed
= false;
508 if (user_input_monitor_
) {
509 size_t current_count
= user_input_monitor_
->GetKeyPressCount();
510 key_pressed
= current_count
!= prev_key_down_count_
;
511 prev_key_down_count_
= current_count
;
512 DVLOG_IF(6, key_pressed
) << "Detected keypress.";
515 // Use SharedMemory and SyncSocket if the client has created a SyncWriter.
516 // Used by all low-latency clients except WebSpeech.
517 if (SharedMemoryAndSyncSocketMode()) {
518 sync_writer_
->Write(source
, volume
, key_pressed
);
519 sync_writer_
->UpdateRecordedBytes(hardware_delay_bytes
);
521 #if defined(AUDIO_POWER_MONITORING)
522 // Only do power-level measurements if DoCreate() has been called. It will
523 // ensure that logging will mainly be done for WebRTC and WebSpeech
525 if (!power_measurement_is_enabled_
)
528 // Perform periodic audio (power) level measurements.
529 if ((base::TimeTicks::Now() - last_audio_level_log_time_
).InSeconds() >
530 kPowerMonitorLogIntervalSeconds
) {
531 // Calculate the average power of the signal, or the energy per sample.
532 const float average_power_dbfs
= AveragePower(*source
);
534 // Add current microphone volume to log and UMA histogram.
535 const int mic_volume_percent
= static_cast<int>(100.0 * volume
);
537 // Use event handler on the audio thread to relay a message to the ARIH
538 // in content which does the actual logging on the IO thread.
539 task_runner_
->PostTask(FROM_HERE
,
540 base::Bind(&AudioInputController::DoLogAudioLevels
,
543 mic_volume_percent
));
545 last_audio_level_log_time_
= base::TimeTicks::Now();
551 // TODO(henrika): Investigate if we can avoid the extra copy here.
552 // (see http://crbug.com/249316 for details). AFAIK, this scope is only
553 // active for WebSpeech clients.
554 scoped_ptr
<AudioBus
> audio_data
=
555 AudioBus::Create(source
->channels(), source
->frames());
556 source
->CopyTo(audio_data
.get());
558 // Ownership of the audio buffer will be with the callback until it is run,
559 // when ownership is passed to the callback function.
560 task_runner_
->PostTask(
563 &AudioInputController::DoOnData
, this, base::Passed(&audio_data
)));
566 void AudioInputController::DoOnData(scoped_ptr
<AudioBus
> data
) {
567 DCHECK(task_runner_
->BelongsToCurrentThread());
569 handler_
->OnData(this, data
.get());
572 void AudioInputController::DoLogAudioLevels(float level_dbfs
,
573 int microphone_volume_percent
) {
574 #if defined(AUDIO_POWER_MONITORING)
575 DCHECK(task_runner_
->BelongsToCurrentThread());
579 // Detect if the user has enabled hardware mute by pressing the mute
580 // button in audio settings for the selected microphone.
581 const bool microphone_is_muted
= stream_
->IsMuted();
582 if (microphone_is_muted
) {
583 LogMicrophoneMuteResult(MICROPHONE_IS_MUTED
);
584 handler_
->OnLog(this, "AIC::OnData: microphone is muted!");
585 // Return early if microphone is muted. No need to adding logs and UMA stats
586 // of audio levels if we know that the micropone is muted.
590 LogMicrophoneMuteResult(MICROPHONE_IS_NOT_MUTED
);
592 std::string log_string
= base::StringPrintf(
593 "AIC::OnData: average audio level=%.2f dBFS", level_dbfs
);
594 static const float kSilenceThresholdDBFS
= -72.24719896f
;
595 if (level_dbfs
< kSilenceThresholdDBFS
)
596 log_string
+= " <=> low audio input level!";
597 handler_
->OnLog(this, log_string
);
599 UpdateSilenceState(level_dbfs
< kSilenceThresholdDBFS
);
601 UMA_HISTOGRAM_PERCENTAGE("Media.MicrophoneVolume", microphone_volume_percent
);
602 log_string
= base::StringPrintf(
603 "AIC::OnData: microphone volume=%d%%", microphone_volume_percent
);
604 if (microphone_volume_percent
< kLowLevelMicrophoneLevelPercent
)
605 log_string
+= " <=> low microphone level!";
606 handler_
->OnLog(this, log_string
);
610 void AudioInputController::OnError(AudioInputStream
* stream
) {
611 // Handle error on the audio-manager thread.
612 task_runner_
->PostTask(FROM_HERE
, base::Bind(
613 &AudioInputController::DoReportError
, this));
616 void AudioInputController::DoStopCloseAndClearStream() {
617 DCHECK(task_runner_
->BelongsToCurrentThread());
619 // Allow calling unconditionally and bail if we don't have a stream to close.
620 if (stream_
!= NULL
) {
626 // The event handler should not be touched after the stream has been closed.
630 void AudioInputController::SetDataIsActive(bool enabled
) {
631 base::subtle::Release_Store(&data_is_active_
, enabled
);
634 bool AudioInputController::GetDataIsActive() {
635 return (base::subtle::Acquire_Load(&data_is_active_
) != false);
638 #if defined(AUDIO_POWER_MONITORING)
639 void AudioInputController::UpdateSilenceState(bool silence
) {
641 if (silence_state_
== SILENCE_STATE_NO_MEASUREMENT
) {
642 silence_state_
= SILENCE_STATE_ONLY_SILENCE
;
643 } else if (silence_state_
== SILENCE_STATE_ONLY_AUDIO
) {
644 silence_state_
= SILENCE_STATE_AUDIO_AND_SILENCE
;
646 DCHECK(silence_state_
== SILENCE_STATE_ONLY_SILENCE
||
647 silence_state_
== SILENCE_STATE_AUDIO_AND_SILENCE
);
650 if (silence_state_
== SILENCE_STATE_NO_MEASUREMENT
) {
651 silence_state_
= SILENCE_STATE_ONLY_AUDIO
;
652 } else if (silence_state_
== SILENCE_STATE_ONLY_SILENCE
) {
653 silence_state_
= SILENCE_STATE_AUDIO_AND_SILENCE
;
655 DCHECK(silence_state_
== SILENCE_STATE_ONLY_AUDIO
||
656 silence_state_
== SILENCE_STATE_AUDIO_AND_SILENCE
);
661 void AudioInputController::LogSilenceState(SilenceState value
) {
662 UMA_HISTOGRAM_ENUMERATION("Media.AudioInputControllerSessionSilenceReport",
664 SILENCE_STATE_MAX
+ 1);