1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_capturer.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "base/strings/stringprintf.h"
12 #include "content/child/child_process.h"
13 #include "content/renderer/media/audio_device_factory.h"
14 #include "content/renderer/media/media_stream_audio_processor.h"
15 #include "content/renderer/media/media_stream_audio_processor_options.h"
16 #include "content/renderer/media/media_stream_audio_source.h"
17 #include "content/renderer/media/webrtc_audio_device_impl.h"
18 #include "content/renderer/media/webrtc_local_audio_track.h"
19 #include "content/renderer/media/webrtc_logging.h"
20 #include "media/audio/sample_rates.h"
26 // Method to check if any of the data in |audio_source| has energy.
27 bool HasDataEnergy(const media::AudioBus
& audio_source
) {
28 for (int ch
= 0; ch
< audio_source
.channels(); ++ch
) {
29 const float* channel_ptr
= audio_source
.channel(ch
);
30 for (int frame
= 0; frame
< audio_source
.frames(); ++frame
) {
31 if (channel_ptr
[frame
] != 0)
36 // All the data is zero.
42 // Reference counted container of WebRtcLocalAudioTrack delegate.
43 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
44 class WebRtcAudioCapturer::TrackOwner
45 : public base::RefCountedThreadSafe
<WebRtcAudioCapturer::TrackOwner
> {
47 explicit TrackOwner(WebRtcLocalAudioTrack
* track
)
50 void Capture(const int16
* audio_data
,
51 base::TimeDelta delay
,
54 bool need_audio_processing
,
55 bool force_report_nonzero_energy
) {
56 base::AutoLock
lock(lock_
);
58 delegate_
->Capture(audio_data
,
62 need_audio_processing
,
63 force_report_nonzero_energy
);
67 void OnSetFormat(const media::AudioParameters
& params
) {
68 base::AutoLock
lock(lock_
);
70 delegate_
->OnSetFormat(params
);
73 void SetAudioProcessor(
74 const scoped_refptr
<MediaStreamAudioProcessor
>& processor
) {
75 base::AutoLock
lock(lock_
);
77 delegate_
->SetAudioProcessor(processor
);
81 base::AutoLock
lock(lock_
);
86 base::AutoLock
lock(lock_
);
89 // This can be reentrant so reset |delegate_| before calling out.
90 WebRtcLocalAudioTrack
* temp
= delegate_
;
95 // Wrapper which allows to use std::find_if() when adding and removing
96 // sinks to/from the list.
98 TrackWrapper(WebRtcLocalAudioTrack
* track
) : track_(track
) {}
100 const scoped_refptr
<WebRtcAudioCapturer::TrackOwner
>& owner
) const {
101 return owner
->IsEqual(track_
);
103 WebRtcLocalAudioTrack
* track_
;
107 virtual ~TrackOwner() {}
110 friend class base::RefCountedThreadSafe
<WebRtcAudioCapturer::TrackOwner
>;
112 bool IsEqual(const WebRtcLocalAudioTrack
* other
) const {
113 base::AutoLock
lock(lock_
);
114 return (other
== delegate_
);
117 // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
118 WebRtcLocalAudioTrack
* delegate_
;
119 mutable base::Lock lock_
;
121 DISALLOW_COPY_AND_ASSIGN(TrackOwner
);
125 scoped_refptr
<WebRtcAudioCapturer
> WebRtcAudioCapturer::CreateCapturer(
126 int render_view_id
, const StreamDeviceInfo
& device_info
,
127 const blink::WebMediaConstraints
& constraints
,
128 WebRtcAudioDeviceImpl
* audio_device
,
129 MediaStreamAudioSource
* audio_source
) {
130 scoped_refptr
<WebRtcAudioCapturer
> capturer
= new WebRtcAudioCapturer(
131 render_view_id
, device_info
, constraints
, audio_device
, audio_source
);
132 if (capturer
->Initialize())
138 bool WebRtcAudioCapturer::Initialize() {
139 DCHECK(thread_checker_
.CalledOnValidThread());
140 DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
141 WebRtcLogMessage(base::StringPrintf(
142 "WAC::Initialize. render_view_id=%d"
143 ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
144 ", session_id=%d, paired_output_sample_rate=%d"
145 ", paired_output_frames_per_buffer=%d, effects=%d. ",
147 device_info_
.device
.input
.channel_layout
,
148 device_info_
.device
.input
.sample_rate
,
149 device_info_
.device
.input
.frames_per_buffer
,
150 device_info_
.session_id
,
151 device_info_
.device
.matched_output
.sample_rate
,
152 device_info_
.device
.matched_output
.frames_per_buffer
,
153 device_info_
.device
.input
.effects
));
155 if (render_view_id_
== -1) {
156 // Return true here to allow injecting a new source via
157 // SetCapturerSourceForTesting() at a later state.
161 MediaAudioConstraints
audio_constraints(constraints_
,
162 device_info_
.device
.input
.effects
);
163 if (!audio_constraints
.IsValid())
166 media::ChannelLayout channel_layout
= static_cast<media::ChannelLayout
>(
167 device_info_
.device
.input
.channel_layout
);
169 // If KEYBOARD_MIC effect is set, change the layout to the corresponding
170 // layout that includes the keyboard mic.
171 if ((device_info_
.device
.input
.effects
&
172 media::AudioParameters::KEYBOARD_MIC
) &&
173 MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() &&
174 audio_constraints
.GetProperty(
175 MediaAudioConstraints::kGoogExperimentalNoiseSuppression
)) {
176 if (channel_layout
== media::CHANNEL_LAYOUT_STEREO
) {
177 channel_layout
= media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC
;
178 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due "
179 << "to KEYBOARD_MIC effect.";
181 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout "
186 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout
;
187 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
188 channel_layout
, media::CHANNEL_LAYOUT_MAX
+ 1);
190 // Verify that the reported input channel configuration is supported.
191 if (channel_layout
!= media::CHANNEL_LAYOUT_MONO
&&
192 channel_layout
!= media::CHANNEL_LAYOUT_STEREO
&&
193 channel_layout
!= media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC
) {
194 DLOG(ERROR
) << channel_layout
195 << " is not a supported input channel configuration.";
199 DVLOG(1) << "Audio input hardware sample rate: "
200 << device_info_
.device
.input
.sample_rate
;
201 media::AudioSampleRate asr
;
202 if (media::ToAudioSampleRate(device_info_
.device
.input
.sample_rate
, &asr
)) {
203 UMA_HISTOGRAM_ENUMERATION(
204 "WebRTC.AudioInputSampleRate", asr
, media::kAudioSampleRateMax
+ 1);
206 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
207 device_info_
.device
.input
.sample_rate
);
210 // Create and configure the default audio capturing source.
211 SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id_
),
213 static_cast<float>(device_info_
.device
.input
.sample_rate
));
215 // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
216 // information from the capturer.
218 audio_device_
->AddAudioCapturer(this);
223 WebRtcAudioCapturer::WebRtcAudioCapturer(
225 const StreamDeviceInfo
& device_info
,
226 const blink::WebMediaConstraints
& constraints
,
227 WebRtcAudioDeviceImpl
* audio_device
,
228 MediaStreamAudioSource
* audio_source
)
229 : constraints_(constraints
),
230 audio_processor_(new rtc::RefCountedObject
<MediaStreamAudioProcessor
>(
232 device_info
.device
.input
.effects
,
235 render_view_id_(render_view_id
),
236 device_info_(device_info
),
238 peer_connection_mode_(false),
240 need_audio_processing_(false),
241 audio_device_(audio_device
),
242 audio_source_(audio_source
) {
243 DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
246 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
247 DCHECK(thread_checker_
.CalledOnValidThread());
248 DCHECK(tracks_
.IsEmpty());
249 DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
253 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack
* track
) {
255 DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
258 base::AutoLock
auto_lock(lock_
);
259 // Verify that |track| is not already added to the list.
260 DCHECK(!tracks_
.Contains(TrackOwner::TrackWrapper(track
)));
262 // Add with a tag, so we remember to call OnSetFormat() on the new
264 scoped_refptr
<TrackOwner
> track_owner(new TrackOwner(track
));
265 tracks_
.AddAndTag(track_owner
.get());
269 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack
* track
) {
270 DCHECK(thread_checker_
.CalledOnValidThread());
271 DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()";
272 bool stop_source
= false;
274 base::AutoLock
auto_lock(lock_
);
276 scoped_refptr
<TrackOwner
> removed_item
=
277 tracks_
.Remove(TrackOwner::TrackWrapper(track
));
279 // Clear the delegate to ensure that no more capture callbacks will
280 // be sent to this sink. Also avoids a possible crash which can happen
281 // if this method is called while capturing is active.
282 if (removed_item
.get()) {
283 removed_item
->Reset();
284 stop_source
= tracks_
.IsEmpty();
288 // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
289 // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
290 // we have to call StopSource on the MediaStreamSource. This will call
291 // MediaStreamAudioSource::DoStopSource which in turn call
292 // WebRtcAudioCapturerer::Stop();
293 audio_source_
->StopSource();
297 void WebRtcAudioCapturer::SetCapturerSource(
298 const scoped_refptr
<media::AudioCapturerSource
>& source
,
299 media::ChannelLayout channel_layout
,
301 DCHECK(thread_checker_
.CalledOnValidThread());
302 DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout
<< ","
303 << "sample_rate=" << sample_rate
<< ")";
304 scoped_refptr
<media::AudioCapturerSource
> old_source
;
306 base::AutoLock
auto_lock(lock_
);
307 if (source_
.get() == source
.get())
310 source_
.swap(old_source
);
313 // Reset the flag to allow starting the new source.
317 DVLOG(1) << "Switching to a new capture source.";
318 if (old_source
.get())
321 // Dispatch the new parameters both to the sink(s) and to the new source,
322 // also apply the new |constraints|.
323 // The idea is to get rid of any dependency of the microphone parameters
324 // which would normally be used by default.
325 // bits_per_sample is always 16 for now.
326 int buffer_size
= GetBufferSize(sample_rate
);
327 media::AudioParameters
params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY
,
328 channel_layout
, sample_rate
,
330 device_info_
.device
.input
.effects
);
333 base::AutoLock
auto_lock(lock_
);
334 // Notify the |audio_processor_| of the new format.
335 audio_processor_
->OnCaptureFormatChanged(params
);
337 MediaAudioConstraints
audio_constraints(constraints_
,
338 device_info_
.device
.input
.effects
);
339 need_audio_processing_
= audio_constraints
.NeedsAudioProcessing();
340 // Notify all tracks about the new format.
345 source
->Initialize(params
, this, session_id());
350 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
351 DCHECK(thread_checker_
.CalledOnValidThread());
352 DVLOG(1) << "EnablePeerConnectionMode";
353 // Do nothing if the peer connection mode has been enabled.
354 if (peer_connection_mode_
)
357 peer_connection_mode_
= true;
358 int render_view_id
= -1;
359 media::AudioParameters input_params
;
361 base::AutoLock
auto_lock(lock_
);
362 // Simply return if there is no existing source or the |render_view_id_| is
364 if (!source_
.get() || render_view_id_
== -1)
367 render_view_id
= render_view_id_
;
368 input_params
= audio_processor_
->InputFormat();
371 // Do nothing if the current buffer size is the WebRtc native buffer size.
372 if (GetBufferSize(input_params
.sample_rate()) ==
373 input_params
.frames_per_buffer()) {
377 // Create a new audio stream as source which will open the hardware using
378 // WebRtc native buffer size.
379 SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id
),
380 input_params
.channel_layout(),
381 static_cast<float>(input_params
.sample_rate()));
384 void WebRtcAudioCapturer::Start() {
385 DCHECK(thread_checker_
.CalledOnValidThread());
386 DVLOG(1) << "WebRtcAudioCapturer::Start()";
387 base::AutoLock
auto_lock(lock_
);
388 if (running_
|| !source_
.get())
391 // Start the data source, i.e., start capturing data from the current source.
392 // We need to set the AGC control before starting the stream.
393 source_
->SetAutomaticGainControl(true);
398 void WebRtcAudioCapturer::Stop() {
399 DCHECK(thread_checker_
.CalledOnValidThread());
400 DVLOG(1) << "WebRtcAudioCapturer::Stop()";
401 scoped_refptr
<media::AudioCapturerSource
> source
;
402 TrackList::ItemList tracks
;
404 base::AutoLock
auto_lock(lock_
);
409 tracks
= tracks_
.Items();
414 // Remove the capturer object from the WebRtcAudioDeviceImpl.
416 audio_device_
->RemoveAudioCapturer(this);
418 for (TrackList::ItemList::const_iterator it
= tracks
.begin();
427 // Stop the audio processor to avoid feeding render data into the processor.
428 audio_processor_
->Stop();
431 void WebRtcAudioCapturer::SetVolume(int volume
) {
432 DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
433 DCHECK_LE(volume
, MaxVolume());
434 double normalized_volume
= static_cast<double>(volume
) / MaxVolume();
435 base::AutoLock
auto_lock(lock_
);
437 source_
->SetVolume(normalized_volume
);
440 int WebRtcAudioCapturer::Volume() const {
441 base::AutoLock
auto_lock(lock_
);
445 int WebRtcAudioCapturer::MaxVolume() const {
446 return WebRtcAudioDeviceImpl::kMaxVolumeLevel
;
449 void WebRtcAudioCapturer::Capture(const media::AudioBus
* audio_source
,
450 int audio_delay_milliseconds
,
453 // This callback is driven by AudioInputDevice::AudioThreadCallback if
454 // |source_| is AudioInputDevice, otherwise it is driven by client's
456 #if defined(OS_WIN) || defined(OS_MACOSX)
457 DCHECK_LE(volume
, 1.0);
458 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
459 // We have a special situation on Linux where the microphone volume can be
460 // "higher than maximum". The input volume slider in the sound preference
461 // allows the user to set a scaling that is higher than 100%. It means that
462 // even if the reported maximum levels is N, the actual microphone level can
463 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
464 DCHECK_LE(volume
, 1.6);
467 TrackList::ItemList tracks
;
468 TrackList::ItemList tracks_to_notify_format
;
469 int current_volume
= 0;
470 base::TimeDelta audio_delay
;
471 bool need_audio_processing
= true;
473 base::AutoLock
auto_lock(lock_
);
477 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
478 // The volume can be higher than 255 on Linux, and it will be cropped to
479 // 255 since AGC does not allow values out of range.
480 volume_
= static_cast<int>((volume
* MaxVolume()) + 0.5);
481 current_volume
= volume_
> MaxVolume() ? MaxVolume() : volume_
;
482 audio_delay
= base::TimeDelta::FromMilliseconds(audio_delay_milliseconds
);
483 audio_delay_
= audio_delay
;
484 key_pressed_
= key_pressed
;
485 tracks
= tracks_
.Items();
486 tracks_
.RetrieveAndClearTags(&tracks_to_notify_format
);
488 // Set the flag to turn on the audio processing in PeerConnection level.
489 // Note that, we turn off the audio processing in PeerConnection if the
490 // processor has already processed the data.
491 need_audio_processing
= need_audio_processing_
?
492 !MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() : false;
495 DCHECK(audio_processor_
->InputFormat().IsValid());
496 DCHECK_EQ(audio_source
->channels(),
497 audio_processor_
->InputFormat().channels());
498 DCHECK_EQ(audio_source
->frames(),
499 audio_processor_
->InputFormat().frames_per_buffer());
501 // Notify the tracks on when the format changes. This will do nothing if
502 // |tracks_to_notify_format| is empty.
503 media::AudioParameters output_params
= audio_processor_
->OutputFormat();
504 for (TrackList::ItemList::const_iterator it
= tracks_to_notify_format
.begin();
505 it
!= tracks_to_notify_format
.end(); ++it
) {
506 (*it
)->OnSetFormat(output_params
);
507 (*it
)->SetAudioProcessor(audio_processor_
);
510 // Figure out if the pre-processed data has any energy or not, the
511 // information will be passed to the track to force the calculator
512 // to report energy in case the post-processed data is zeroed by the audio
514 const bool force_report_nonzero_energy
= HasDataEnergy(*audio_source
);
516 // Push the data to the processor for processing.
517 audio_processor_
->PushCaptureData(audio_source
);
519 // Process and consume the data in the processor until there is not enough
520 // data in the processor.
521 int16
* output
= NULL
;
523 while (audio_processor_
->ProcessAndConsumeData(
524 audio_delay
, current_volume
, key_pressed
, &new_volume
, &output
)) {
525 // Feed the post-processed data to the tracks.
526 for (TrackList::ItemList::const_iterator it
= tracks
.begin();
527 it
!= tracks
.end(); ++it
) {
528 (*it
)->Capture(output
, audio_delay
, current_volume
, key_pressed
,
529 need_audio_processing
, force_report_nonzero_energy
);
533 SetVolume(new_volume
);
535 // Update the |current_volume| to avoid passing the old volume to AGC.
536 current_volume
= new_volume
;
541 void WebRtcAudioCapturer::OnCaptureError() {
545 media::AudioParameters
WebRtcAudioCapturer::source_audio_parameters() const {
546 base::AutoLock
auto_lock(lock_
);
547 return audio_processor_
.get() ? audio_processor_
->InputFormat()
548 : media::AudioParameters();
551 bool WebRtcAudioCapturer::GetPairedOutputParameters(
553 int* output_sample_rate
,
554 int* output_frames_per_buffer
) const {
555 // Don't set output parameters unless all of them are valid.
556 if (device_info_
.session_id
<= 0 ||
557 !device_info_
.device
.matched_output
.sample_rate
||
558 !device_info_
.device
.matched_output
.frames_per_buffer
)
561 *session_id
= device_info_
.session_id
;
562 *output_sample_rate
= device_info_
.device
.matched_output
.sample_rate
;
563 *output_frames_per_buffer
=
564 device_info_
.device
.matched_output
.frames_per_buffer
;
569 int WebRtcAudioCapturer::GetBufferSize(int sample_rate
) const {
570 DCHECK(thread_checker_
.CalledOnValidThread());
571 #if defined(OS_ANDROID)
572 // TODO(henrika): Tune and adjust buffer size on Android.
573 return (2 * sample_rate
/ 100);
576 // PeerConnection is running at a buffer size of 10ms data. A multiple of
577 // 10ms as the buffer size can give the best performance to PeerConnection.
578 int peer_connection_buffer_size
= sample_rate
/ 100;
580 // Use the native hardware buffer size in non peer connection mode when the
581 // platform is using a native buffer size smaller than the PeerConnection
583 int hardware_buffer_size
= device_info_
.device
.input
.frames_per_buffer
;
584 if (!peer_connection_mode_
&& hardware_buffer_size
&&
585 hardware_buffer_size
<= peer_connection_buffer_size
) {
586 return hardware_buffer_size
;
589 return (sample_rate
/ 100);
592 void WebRtcAudioCapturer::GetAudioProcessingParams(
593 base::TimeDelta
* delay
, int* volume
, bool* key_pressed
) {
594 base::AutoLock
auto_lock(lock_
);
595 *delay
= audio_delay_
;
597 *key_pressed
= key_pressed_
;
600 void WebRtcAudioCapturer::SetCapturerSourceForTesting(
601 const scoped_refptr
<media::AudioCapturerSource
>& source
,
602 media::AudioParameters params
) {
603 // Create a new audio stream as source which uses the new source.
604 SetCapturerSource(source
, params
.channel_layout(),
605 static_cast<float>(params
.sample_rate()));
608 } // namespace content