1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_capturer.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "base/strings/stringprintf.h"
12 #include "content/child/child_process.h"
13 #include "content/renderer/media/audio_device_factory.h"
14 #include "content/renderer/media/media_stream_audio_processor.h"
15 #include "content/renderer/media/media_stream_audio_processor_options.h"
16 #include "content/renderer/media/media_stream_audio_source.h"
17 #include "content/renderer/media/media_stream_constraints_util.h"
18 #include "content/renderer/media/webrtc_audio_device_impl.h"
19 #include "content/renderer/media/webrtc_local_audio_track.h"
20 #include "content/renderer/media/webrtc_logging.h"
21 #include "media/audio/sample_rates.h"
27 // Audio buffer sizes are specified in milliseconds.
28 const char kAudioLatency
[] = "latencyMs";
29 const int kMinAudioLatencyMs
= 0;
30 const int kMaxAudioLatencyMs
= 10000;
32 // Method to check if any of the data in |audio_source| has energy.
33 bool HasDataEnergy(const media::AudioBus
& audio_source
) {
34 for (int ch
= 0; ch
< audio_source
.channels(); ++ch
) {
35 const float* channel_ptr
= audio_source
.channel(ch
);
36 for (int frame
= 0; frame
< audio_source
.frames(); ++frame
) {
37 if (channel_ptr
[frame
] != 0)
42 // All the data is zero.
48 // Reference counted container of WebRtcLocalAudioTrack delegate.
49 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
50 class WebRtcAudioCapturer::TrackOwner
51 : public base::RefCountedThreadSafe
<WebRtcAudioCapturer::TrackOwner
> {
53 explicit TrackOwner(WebRtcLocalAudioTrack
* track
)
56 void Capture(const media::AudioBus
& audio_bus
,
57 base::TimeTicks estimated_capture_time
,
58 bool force_report_nonzero_energy
) {
59 base::AutoLock
lock(lock_
);
61 delegate_
->Capture(audio_bus
,
62 estimated_capture_time
,
63 force_report_nonzero_energy
);
67 void OnSetFormat(const media::AudioParameters
& params
) {
68 base::AutoLock
lock(lock_
);
70 delegate_
->OnSetFormat(params
);
73 void SetAudioProcessor(
74 const scoped_refptr
<MediaStreamAudioProcessor
>& processor
) {
75 base::AutoLock
lock(lock_
);
77 delegate_
->SetAudioProcessor(processor
);
81 base::AutoLock
lock(lock_
);
86 base::AutoLock
lock(lock_
);
89 // This can be reentrant so reset |delegate_| before calling out.
90 WebRtcLocalAudioTrack
* temp
= delegate_
;
95 // Wrapper which allows to use std::find_if() when adding and removing
96 // sinks to/from the list.
98 explicit TrackWrapper(WebRtcLocalAudioTrack
* track
) : track_(track
) {}
100 const scoped_refptr
<WebRtcAudioCapturer::TrackOwner
>& owner
) const {
101 return owner
->IsEqual(track_
);
103 WebRtcLocalAudioTrack
* track_
;
107 virtual ~TrackOwner() {}
110 friend class base::RefCountedThreadSafe
<WebRtcAudioCapturer::TrackOwner
>;
112 bool IsEqual(const WebRtcLocalAudioTrack
* other
) const {
113 base::AutoLock
lock(lock_
);
114 return (other
== delegate_
);
117 // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
118 WebRtcLocalAudioTrack
* delegate_
;
119 mutable base::Lock lock_
;
121 DISALLOW_COPY_AND_ASSIGN(TrackOwner
);
125 scoped_refptr
<WebRtcAudioCapturer
> WebRtcAudioCapturer::CreateCapturer(
127 const StreamDeviceInfo
& device_info
,
128 const blink::WebMediaConstraints
& constraints
,
129 WebRtcAudioDeviceImpl
* audio_device
,
130 MediaStreamAudioSource
* audio_source
) {
131 scoped_refptr
<WebRtcAudioCapturer
> capturer
= new WebRtcAudioCapturer(
132 render_frame_id
, device_info
, constraints
, audio_device
, audio_source
);
133 if (capturer
->Initialize())
139 bool WebRtcAudioCapturer::Initialize() {
140 DCHECK(thread_checker_
.CalledOnValidThread());
141 DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
142 WebRtcLogMessage(base::StringPrintf(
143 "WAC::Initialize. render_frame_id=%d"
144 ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
145 ", session_id=%d, paired_output_sample_rate=%d"
146 ", paired_output_frames_per_buffer=%d, effects=%d. ",
147 render_frame_id_
, device_info_
.device
.input
.channel_layout
,
148 device_info_
.device
.input
.sample_rate
,
149 device_info_
.device
.input
.frames_per_buffer
, device_info_
.session_id
,
150 device_info_
.device
.matched_output
.sample_rate
,
151 device_info_
.device
.matched_output
.frames_per_buffer
,
152 device_info_
.device
.input
.effects
));
154 if (render_frame_id_
== -1) {
155 // Return true here to allow injecting a new source via
156 // SetCapturerSourceForTesting() at a later state.
160 MediaAudioConstraints
audio_constraints(constraints_
,
161 device_info_
.device
.input
.effects
);
162 if (!audio_constraints
.IsValid())
165 media::ChannelLayout channel_layout
= static_cast<media::ChannelLayout
>(
166 device_info_
.device
.input
.channel_layout
);
168 // If KEYBOARD_MIC effect is set, change the layout to the corresponding
169 // layout that includes the keyboard mic.
170 if ((device_info_
.device
.input
.effects
&
171 media::AudioParameters::KEYBOARD_MIC
) &&
172 audio_constraints
.GetProperty(
173 MediaAudioConstraints::kGoogExperimentalNoiseSuppression
)) {
174 if (channel_layout
== media::CHANNEL_LAYOUT_STEREO
) {
175 channel_layout
= media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC
;
176 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due "
177 << "to KEYBOARD_MIC effect.";
179 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout "
184 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout
;
185 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
186 channel_layout
, media::CHANNEL_LAYOUT_MAX
+ 1);
188 // Verify that the reported input channel configuration is supported.
189 if (channel_layout
!= media::CHANNEL_LAYOUT_MONO
&&
190 channel_layout
!= media::CHANNEL_LAYOUT_STEREO
&&
191 channel_layout
!= media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC
) {
192 DLOG(ERROR
) << channel_layout
193 << " is not a supported input channel configuration.";
197 DVLOG(1) << "Audio input hardware sample rate: "
198 << device_info_
.device
.input
.sample_rate
;
199 media::AudioSampleRate asr
;
200 if (media::ToAudioSampleRate(device_info_
.device
.input
.sample_rate
, &asr
)) {
201 UMA_HISTOGRAM_ENUMERATION(
202 "WebRTC.AudioInputSampleRate", asr
, media::kAudioSampleRateMax
+ 1);
204 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
205 device_info_
.device
.input
.sample_rate
);
208 // Initialize the buffer size to zero, which means it wasn't specified.
209 // If it is out of range, we return it to zero.
210 int buffer_size_ms
= 0;
211 int buffer_size_samples
= 0;
212 GetConstraintValueAsInteger(constraints_
, kAudioLatency
, &buffer_size_ms
);
213 if (buffer_size_ms
< kMinAudioLatencyMs
||
214 buffer_size_ms
> kMaxAudioLatencyMs
) {
215 DVLOG(1) << "Ignoring out of range buffer size " << buffer_size_ms
;
217 buffer_size_samples
=
218 device_info_
.device
.input
.sample_rate
* buffer_size_ms
/ 1000;
220 DVLOG_IF(1, buffer_size_samples
> 0)
221 << "Custom audio buffer size: " << buffer_size_samples
<< " samples";
223 // Create and configure the default audio capturing source.
224 SetCapturerSourceInternal(
225 AudioDeviceFactory::NewInputDevice(render_frame_id_
),
227 device_info_
.device
.input
.sample_rate
,
228 buffer_size_samples
);
230 // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
231 // information from the capturer.
233 audio_device_
->AddAudioCapturer(this);
238 WebRtcAudioCapturer::WebRtcAudioCapturer(
240 const StreamDeviceInfo
& device_info
,
241 const blink::WebMediaConstraints
& constraints
,
242 WebRtcAudioDeviceImpl
* audio_device
,
243 MediaStreamAudioSource
* audio_source
)
244 : constraints_(constraints
),
245 audio_processor_(new rtc::RefCountedObject
<MediaStreamAudioProcessor
>(
247 device_info
.device
.input
,
250 render_frame_id_(render_frame_id
),
251 device_info_(device_info
),
253 peer_connection_mode_(false),
254 audio_device_(audio_device
),
255 audio_source_(audio_source
) {
256 DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
259 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
260 DCHECK(thread_checker_
.CalledOnValidThread());
261 DCHECK(tracks_
.IsEmpty());
262 DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
266 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack
* track
) {
268 DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
271 base::AutoLock
auto_lock(lock_
);
272 // Verify that |track| is not already added to the list.
273 DCHECK(!tracks_
.Contains(TrackOwner::TrackWrapper(track
)));
275 // Add with a tag, so we remember to call OnSetFormat() on the new
277 scoped_refptr
<TrackOwner
> track_owner(new TrackOwner(track
));
278 tracks_
.AddAndTag(track_owner
.get());
282 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack
* track
) {
283 DCHECK(thread_checker_
.CalledOnValidThread());
284 DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()";
285 bool stop_source
= false;
287 base::AutoLock
auto_lock(lock_
);
289 scoped_refptr
<TrackOwner
> removed_item
=
290 tracks_
.Remove(TrackOwner::TrackWrapper(track
));
292 // Clear the delegate to ensure that no more capture callbacks will
293 // be sent to this sink. Also avoids a possible crash which can happen
294 // if this method is called while capturing is active.
295 if (removed_item
.get()) {
296 removed_item
->Reset();
297 stop_source
= tracks_
.IsEmpty();
301 // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
302 // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
303 // we have to call StopSource on the MediaStreamSource. This will call
304 // MediaStreamAudioSource::DoStopSource which in turn call
305 // WebRtcAudioCapturerer::Stop();
306 audio_source_
->StopSource();
310 void WebRtcAudioCapturer::SetCapturerSourceInternal(
311 const scoped_refptr
<media::AudioCapturerSource
>& source
,
312 media::ChannelLayout channel_layout
,
315 DCHECK(thread_checker_
.CalledOnValidThread());
316 DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout
<< ","
317 << "sample_rate=" << sample_rate
<< ")";
318 scoped_refptr
<media::AudioCapturerSource
> old_source
;
320 base::AutoLock
auto_lock(lock_
);
321 if (source_
.get() == source
.get())
324 source_
.swap(old_source
);
327 // Reset the flag to allow starting the new source.
331 DVLOG(1) << "Switching to a new capture source.";
332 if (old_source
.get())
335 // If the buffer size is zero, it has not been specified.
336 // We either default to 10ms, or use the hardware buffer size.
337 if (buffer_size
== 0)
338 buffer_size
= GetBufferSize(sample_rate
);
340 // Dispatch the new parameters both to the sink(s) and to the new source,
341 // also apply the new |constraints|.
342 // The idea is to get rid of any dependency of the microphone parameters
343 // which would normally be used by default.
344 // bits_per_sample is always 16 for now.
345 media::AudioParameters
params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY
,
346 channel_layout
, sample_rate
, 16, buffer_size
);
347 params
.set_effects(device_info_
.device
.input
.effects
);
350 base::AutoLock
auto_lock(lock_
);
351 // Notify the |audio_processor_| of the new format.
352 audio_processor_
->OnCaptureFormatChanged(params
);
354 // Notify all tracks about the new format.
359 source
->Initialize(params
, this, session_id());
364 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
365 DCHECK(thread_checker_
.CalledOnValidThread());
366 DVLOG(1) << "EnablePeerConnectionMode";
367 // Do nothing if the peer connection mode has been enabled.
368 if (peer_connection_mode_
)
371 peer_connection_mode_
= true;
372 int render_frame_id
= -1;
373 media::AudioParameters input_params
;
375 base::AutoLock
auto_lock(lock_
);
376 // Simply return if there is no existing source or the |render_frame_id_| is
378 if (!source_
.get() || render_frame_id_
== -1)
381 render_frame_id
= render_frame_id_
;
382 input_params
= audio_processor_
->InputFormat();
385 // Do nothing if the current buffer size is the WebRtc native buffer size.
386 if (GetBufferSize(input_params
.sample_rate()) ==
387 input_params
.frames_per_buffer()) {
391 // Create a new audio stream as source which will open the hardware using
392 // WebRtc native buffer size.
393 SetCapturerSourceInternal(AudioDeviceFactory::NewInputDevice(render_frame_id
),
394 input_params
.channel_layout(),
395 input_params
.sample_rate(),
399 void WebRtcAudioCapturer::Start() {
400 DCHECK(thread_checker_
.CalledOnValidThread());
401 DVLOG(1) << "WebRtcAudioCapturer::Start()";
402 base::AutoLock
auto_lock(lock_
);
403 if (running_
|| !source_
.get())
406 // Start the data source, i.e., start capturing data from the current source.
407 // We need to set the AGC control before starting the stream.
408 source_
->SetAutomaticGainControl(true);
413 void WebRtcAudioCapturer::Stop() {
414 DCHECK(thread_checker_
.CalledOnValidThread());
415 DVLOG(1) << "WebRtcAudioCapturer::Stop()";
416 scoped_refptr
<media::AudioCapturerSource
> source
;
417 TrackList::ItemList tracks
;
419 base::AutoLock
auto_lock(lock_
);
424 tracks
= tracks_
.Items();
429 // Remove the capturer object from the WebRtcAudioDeviceImpl.
431 audio_device_
->RemoveAudioCapturer(this);
433 for (TrackList::ItemList::const_iterator it
= tracks
.begin();
442 // Stop the audio processor to avoid feeding render data into the processor.
443 audio_processor_
->Stop();
446 void WebRtcAudioCapturer::SetVolume(int volume
) {
447 DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
448 DCHECK_LE(volume
, MaxVolume());
449 double normalized_volume
= static_cast<double>(volume
) / MaxVolume();
450 base::AutoLock
auto_lock(lock_
);
452 source_
->SetVolume(normalized_volume
);
455 int WebRtcAudioCapturer::Volume() const {
456 base::AutoLock
auto_lock(lock_
);
460 int WebRtcAudioCapturer::MaxVolume() const {
461 return WebRtcAudioDeviceImpl::kMaxVolumeLevel
;
464 media::AudioParameters
WebRtcAudioCapturer::GetOutputFormat() const {
465 DCHECK(thread_checker_
.CalledOnValidThread());
466 return audio_processor_
->OutputFormat();
469 void WebRtcAudioCapturer::Capture(const media::AudioBus
* audio_source
,
470 int audio_delay_milliseconds
,
473 // This callback is driven by AudioInputDevice::AudioThreadCallback if
474 // |source_| is AudioInputDevice, otherwise it is driven by client's
476 #if defined(OS_WIN) || defined(OS_MACOSX)
477 DCHECK_LE(volume
, 1.0);
478 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
479 // We have a special situation on Linux where the microphone volume can be
480 // "higher than maximum". The input volume slider in the sound preference
481 // allows the user to set a scaling that is higher than 100%. It means that
482 // even if the reported maximum levels is N, the actual microphone level can
483 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
484 DCHECK_LE(volume
, 1.6);
487 // TODO(miu): Plumbing is needed to determine the actual capture timestamp
488 // of the audio, instead of just snapshotting TimeTicks::Now(), for proper
489 // audio/video sync. http://crbug.com/335335
490 const base::TimeTicks reference_clock_snapshot
= base::TimeTicks::Now();
492 TrackList::ItemList tracks
;
493 TrackList::ItemList tracks_to_notify_format
;
494 int current_volume
= 0;
496 base::AutoLock
auto_lock(lock_
);
500 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
501 // The volume can be higher than 255 on Linux, and it will be cropped to
502 // 255 since AGC does not allow values out of range.
503 volume_
= static_cast<int>((volume
* MaxVolume()) + 0.5);
504 current_volume
= volume_
> MaxVolume() ? MaxVolume() : volume_
;
505 tracks
= tracks_
.Items();
506 tracks_
.RetrieveAndClearTags(&tracks_to_notify_format
);
509 DCHECK(audio_processor_
->InputFormat().IsValid());
510 DCHECK_EQ(audio_source
->channels(),
511 audio_processor_
->InputFormat().channels());
512 DCHECK_EQ(audio_source
->frames(),
513 audio_processor_
->InputFormat().frames_per_buffer());
515 // Notify the tracks on when the format changes. This will do nothing if
516 // |tracks_to_notify_format| is empty.
517 const media::AudioParameters
& output_params
=
518 audio_processor_
->OutputFormat();
519 for (const auto& track
: tracks_to_notify_format
) {
520 track
->OnSetFormat(output_params
);
521 track
->SetAudioProcessor(audio_processor_
);
524 // Figure out if the pre-processed data has any energy or not, the
525 // information will be passed to the track to force the calculator
526 // to report energy in case the post-processed data is zeroed by the audio
528 const bool force_report_nonzero_energy
= HasDataEnergy(*audio_source
);
530 // Push the data to the processor for processing.
531 audio_processor_
->PushCaptureData(
533 base::TimeDelta::FromMilliseconds(audio_delay_milliseconds
));
535 // Process and consume the data in the processor until there is not enough
536 // data in the processor.
537 media::AudioBus
* processed_data
= nullptr;
538 base::TimeDelta processed_data_audio_delay
;
540 while (audio_processor_
->ProcessAndConsumeData(
541 current_volume
, key_pressed
,
542 &processed_data
, &processed_data_audio_delay
, &new_volume
)) {
543 DCHECK(processed_data
);
544 const base::TimeTicks processed_data_capture_time
=
545 reference_clock_snapshot
- processed_data_audio_delay
;
546 for (const auto& track
: tracks
) {
547 track
->Capture(*processed_data
,
548 processed_data_capture_time
,
549 force_report_nonzero_energy
);
553 SetVolume(new_volume
);
555 // Update the |current_volume| to avoid passing the old volume to AGC.
556 current_volume
= new_volume
;
561 void WebRtcAudioCapturer::OnCaptureError(const std::string
& message
) {
562 WebRtcLogMessage("WAC::OnCaptureError: " + message
);
565 media::AudioParameters
WebRtcAudioCapturer::source_audio_parameters() const {
566 base::AutoLock
auto_lock(lock_
);
567 return audio_processor_
.get() ? audio_processor_
->InputFormat()
568 : media::AudioParameters();
571 bool WebRtcAudioCapturer::GetPairedOutputParameters(
573 int* output_sample_rate
,
574 int* output_frames_per_buffer
) const {
575 // Don't set output parameters unless all of them are valid.
576 if (device_info_
.session_id
<= 0 ||
577 !device_info_
.device
.matched_output
.sample_rate
||
578 !device_info_
.device
.matched_output
.frames_per_buffer
)
581 *session_id
= device_info_
.session_id
;
582 *output_sample_rate
= device_info_
.device
.matched_output
.sample_rate
;
583 *output_frames_per_buffer
=
584 device_info_
.device
.matched_output
.frames_per_buffer
;
589 int WebRtcAudioCapturer::GetBufferSize(int sample_rate
) const {
590 DCHECK(thread_checker_
.CalledOnValidThread());
591 #if defined(OS_ANDROID)
592 // TODO(henrika): Tune and adjust buffer size on Android.
593 return (2 * sample_rate
/ 100);
596 // PeerConnection is running at a buffer size of 10ms data. A multiple of
597 // 10ms as the buffer size can give the best performance to PeerConnection.
598 int peer_connection_buffer_size
= sample_rate
/ 100;
600 // Use the native hardware buffer size in non peer connection mode when the
601 // platform is using a native buffer size smaller than the PeerConnection
602 // buffer size and audio processing is off.
603 int hardware_buffer_size
= device_info_
.device
.input
.frames_per_buffer
;
604 if (!peer_connection_mode_
&& hardware_buffer_size
&&
605 hardware_buffer_size
<= peer_connection_buffer_size
&&
606 !audio_processor_
->has_audio_processing()) {
607 DVLOG(1) << "WebRtcAudioCapturer is using hardware buffer size "
608 << hardware_buffer_size
;
609 return hardware_buffer_size
;
612 return (sample_rate
/ 100);
615 void WebRtcAudioCapturer::SetCapturerSource(
616 const scoped_refptr
<media::AudioCapturerSource
>& source
,
617 media::AudioParameters params
) {
618 // Create a new audio stream as source which uses the new source.
619 SetCapturerSourceInternal(source
,
620 params
.channel_layout(),
621 params
.sample_rate(),
625 } // namespace content