Roll src/third_party/WebKit d10c917:a1123a1 (svn 198729:198730)
[chromium-blink-merge.git] / content / renderer / media / webrtc_audio_capturer.cc
blobdf46265d8fb519f93c5ca69bd1de88851467fbe2
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_capturer.h"
7 #include "base/bind.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "base/strings/stringprintf.h"
12 #include "content/child/child_process.h"
13 #include "content/renderer/media/audio_device_factory.h"
14 #include "content/renderer/media/media_stream_audio_processor.h"
15 #include "content/renderer/media/media_stream_audio_processor_options.h"
16 #include "content/renderer/media/media_stream_audio_source.h"
17 #include "content/renderer/media/media_stream_constraints_util.h"
18 #include "content/renderer/media/webrtc_audio_device_impl.h"
19 #include "content/renderer/media/webrtc_local_audio_track.h"
20 #include "content/renderer/media/webrtc_logging.h"
21 #include "media/audio/sample_rates.h"
23 namespace content {
25 namespace {
27 // Audio buffer sizes are specified in milliseconds.
28 const char kAudioLatency[] = "latencyMs";
29 const int kMinAudioLatencyMs = 0;
30 const int kMaxAudioLatencyMs = 10000;
32 // Method to check if any of the data in |audio_source| has energy.
33 bool HasDataEnergy(const media::AudioBus& audio_source) {
34 for (int ch = 0; ch < audio_source.channels(); ++ch) {
35 const float* channel_ptr = audio_source.channel(ch);
36 for (int frame = 0; frame < audio_source.frames(); ++frame) {
37 if (channel_ptr[frame] != 0)
38 return true;
42 // All the data is zero.
43 return false;
46 } // namespace
48 // Reference counted container of WebRtcLocalAudioTrack delegate.
49 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
50 class WebRtcAudioCapturer::TrackOwner
51 : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> {
52 public:
53 explicit TrackOwner(WebRtcLocalAudioTrack* track)
54 : delegate_(track) {}
56 void Capture(const media::AudioBus& audio_bus,
57 base::TimeTicks estimated_capture_time,
58 bool force_report_nonzero_energy) {
59 base::AutoLock lock(lock_);
60 if (delegate_) {
61 delegate_->Capture(audio_bus,
62 estimated_capture_time,
63 force_report_nonzero_energy);
67 void OnSetFormat(const media::AudioParameters& params) {
68 base::AutoLock lock(lock_);
69 if (delegate_)
70 delegate_->OnSetFormat(params);
73 void SetAudioProcessor(
74 const scoped_refptr<MediaStreamAudioProcessor>& processor) {
75 base::AutoLock lock(lock_);
76 if (delegate_)
77 delegate_->SetAudioProcessor(processor);
80 void Reset() {
81 base::AutoLock lock(lock_);
82 delegate_ = NULL;
85 void Stop() {
86 base::AutoLock lock(lock_);
87 DCHECK(delegate_);
89 // This can be reentrant so reset |delegate_| before calling out.
90 WebRtcLocalAudioTrack* temp = delegate_;
91 delegate_ = NULL;
92 temp->Stop();
95 // Wrapper which allows to use std::find_if() when adding and removing
96 // sinks to/from the list.
97 struct TrackWrapper {
98 explicit TrackWrapper(WebRtcLocalAudioTrack* track) : track_(track) {}
99 bool operator()(
100 const scoped_refptr<WebRtcAudioCapturer::TrackOwner>& owner) const {
101 return owner->IsEqual(track_);
103 WebRtcLocalAudioTrack* track_;
106 protected:
107 virtual ~TrackOwner() {}
109 private:
110 friend class base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner>;
112 bool IsEqual(const WebRtcLocalAudioTrack* other) const {
113 base::AutoLock lock(lock_);
114 return (other == delegate_);
117 // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
118 WebRtcLocalAudioTrack* delegate_;
119 mutable base::Lock lock_;
121 DISALLOW_COPY_AND_ASSIGN(TrackOwner);
124 // static
125 scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
126 int render_frame_id,
127 const StreamDeviceInfo& device_info,
128 const blink::WebMediaConstraints& constraints,
129 WebRtcAudioDeviceImpl* audio_device,
130 MediaStreamAudioSource* audio_source) {
131 scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
132 render_frame_id, device_info, constraints, audio_device, audio_source);
133 if (capturer->Initialize())
134 return capturer;
136 return NULL;
139 bool WebRtcAudioCapturer::Initialize() {
140 DCHECK(thread_checker_.CalledOnValidThread());
141 DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
142 WebRtcLogMessage(base::StringPrintf(
143 "WAC::Initialize. render_frame_id=%d"
144 ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
145 ", session_id=%d, paired_output_sample_rate=%d"
146 ", paired_output_frames_per_buffer=%d, effects=%d. ",
147 render_frame_id_, device_info_.device.input.channel_layout,
148 device_info_.device.input.sample_rate,
149 device_info_.device.input.frames_per_buffer, device_info_.session_id,
150 device_info_.device.matched_output.sample_rate,
151 device_info_.device.matched_output.frames_per_buffer,
152 device_info_.device.input.effects));
154 if (render_frame_id_ == -1) {
155 // Return true here to allow injecting a new source via
156 // SetCapturerSourceForTesting() at a later state.
157 return true;
160 MediaAudioConstraints audio_constraints(constraints_,
161 device_info_.device.input.effects);
162 if (!audio_constraints.IsValid())
163 return false;
165 media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>(
166 device_info_.device.input.channel_layout);
168 // If KEYBOARD_MIC effect is set, change the layout to the corresponding
169 // layout that includes the keyboard mic.
170 if ((device_info_.device.input.effects &
171 media::AudioParameters::KEYBOARD_MIC) &&
172 audio_constraints.GetProperty(
173 MediaAudioConstraints::kGoogExperimentalNoiseSuppression)) {
174 if (channel_layout == media::CHANNEL_LAYOUT_STEREO) {
175 channel_layout = media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC;
176 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due "
177 << "to KEYBOARD_MIC effect.";
178 } else {
179 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout "
180 << channel_layout;
184 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout;
185 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
186 channel_layout, media::CHANNEL_LAYOUT_MAX + 1);
188 // Verify that the reported input channel configuration is supported.
189 if (channel_layout != media::CHANNEL_LAYOUT_MONO &&
190 channel_layout != media::CHANNEL_LAYOUT_STEREO &&
191 channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) {
192 DLOG(ERROR) << channel_layout
193 << " is not a supported input channel configuration.";
194 return false;
197 DVLOG(1) << "Audio input hardware sample rate: "
198 << device_info_.device.input.sample_rate;
199 media::AudioSampleRate asr;
200 if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) {
201 UMA_HISTOGRAM_ENUMERATION(
202 "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
203 } else {
204 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
205 device_info_.device.input.sample_rate);
208 // Initialize the buffer size to zero, which means it wasn't specified.
209 // If it is out of range, we return it to zero.
210 int buffer_size_ms = 0;
211 int buffer_size_samples = 0;
212 GetConstraintValueAsInteger(constraints_, kAudioLatency, &buffer_size_ms);
213 if (buffer_size_ms < kMinAudioLatencyMs ||
214 buffer_size_ms > kMaxAudioLatencyMs) {
215 DVLOG(1) << "Ignoring out of range buffer size " << buffer_size_ms;
216 } else {
217 buffer_size_samples =
218 device_info_.device.input.sample_rate * buffer_size_ms / 1000;
220 DVLOG_IF(1, buffer_size_samples > 0)
221 << "Custom audio buffer size: " << buffer_size_samples << " samples";
223 // Create and configure the default audio capturing source.
224 SetCapturerSourceInternal(
225 AudioDeviceFactory::NewInputDevice(render_frame_id_),
226 channel_layout,
227 device_info_.device.input.sample_rate,
228 buffer_size_samples);
230 // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
231 // information from the capturer.
232 if (audio_device_)
233 audio_device_->AddAudioCapturer(this);
235 return true;
238 WebRtcAudioCapturer::WebRtcAudioCapturer(
239 int render_frame_id,
240 const StreamDeviceInfo& device_info,
241 const blink::WebMediaConstraints& constraints,
242 WebRtcAudioDeviceImpl* audio_device,
243 MediaStreamAudioSource* audio_source)
244 : constraints_(constraints),
245 audio_processor_(new rtc::RefCountedObject<MediaStreamAudioProcessor>(
246 constraints,
247 device_info.device.input.effects,
248 audio_device)),
249 running_(false),
250 render_frame_id_(render_frame_id),
251 device_info_(device_info),
252 volume_(0),
253 peer_connection_mode_(false),
254 audio_device_(audio_device),
255 audio_source_(audio_source) {
256 DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
259 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
260 DCHECK(thread_checker_.CalledOnValidThread());
261 DCHECK(tracks_.IsEmpty());
262 DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
263 Stop();
266 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
267 DCHECK(track);
268 DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
271 base::AutoLock auto_lock(lock_);
272 // Verify that |track| is not already added to the list.
273 DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track)));
275 // Add with a tag, so we remember to call OnSetFormat() on the new
276 // track.
277 scoped_refptr<TrackOwner> track_owner(new TrackOwner(track));
278 tracks_.AddAndTag(track_owner.get());
282 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack* track) {
283 DCHECK(thread_checker_.CalledOnValidThread());
284 DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()";
285 bool stop_source = false;
287 base::AutoLock auto_lock(lock_);
289 scoped_refptr<TrackOwner> removed_item =
290 tracks_.Remove(TrackOwner::TrackWrapper(track));
292 // Clear the delegate to ensure that no more capture callbacks will
293 // be sent to this sink. Also avoids a possible crash which can happen
294 // if this method is called while capturing is active.
295 if (removed_item.get()) {
296 removed_item->Reset();
297 stop_source = tracks_.IsEmpty();
300 if (stop_source) {
301 // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
302 // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
303 // we have to call StopSource on the MediaStreamSource. This will call
304 // MediaStreamAudioSource::DoStopSource which in turn call
305 // WebRtcAudioCapturerer::Stop();
306 audio_source_->StopSource();
310 void WebRtcAudioCapturer::SetCapturerSourceInternal(
311 const scoped_refptr<media::AudioCapturerSource>& source,
312 media::ChannelLayout channel_layout,
313 int sample_rate,
314 int buffer_size) {
315 DCHECK(thread_checker_.CalledOnValidThread());
316 DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
317 << "sample_rate=" << sample_rate << ")";
318 scoped_refptr<media::AudioCapturerSource> old_source;
320 base::AutoLock auto_lock(lock_);
321 if (source_.get() == source.get())
322 return;
324 source_.swap(old_source);
325 source_ = source;
327 // Reset the flag to allow starting the new source.
328 running_ = false;
331 DVLOG(1) << "Switching to a new capture source.";
332 if (old_source.get())
333 old_source->Stop();
335 // If the buffer size is zero, it has not been specified.
336 // We either default to 10ms, or use the hardware buffer size.
337 if (buffer_size == 0)
338 buffer_size = GetBufferSize(sample_rate);
340 // Dispatch the new parameters both to the sink(s) and to the new source,
341 // also apply the new |constraints|.
342 // The idea is to get rid of any dependency of the microphone parameters
343 // which would normally be used by default.
344 // bits_per_sample is always 16 for now.
345 media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
346 channel_layout,
347 sample_rate,
349 buffer_size,
350 device_info_.device.input.effects);
353 base::AutoLock auto_lock(lock_);
354 // Notify the |audio_processor_| of the new format.
355 audio_processor_->OnCaptureFormatChanged(params);
357 // Notify all tracks about the new format.
358 tracks_.TagAll();
361 if (source.get())
362 source->Initialize(params, this, session_id());
364 Start();
367 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
368 DCHECK(thread_checker_.CalledOnValidThread());
369 DVLOG(1) << "EnablePeerConnectionMode";
370 // Do nothing if the peer connection mode has been enabled.
371 if (peer_connection_mode_)
372 return;
374 peer_connection_mode_ = true;
375 int render_frame_id = -1;
376 media::AudioParameters input_params;
378 base::AutoLock auto_lock(lock_);
379 // Simply return if there is no existing source or the |render_frame_id_| is
380 // not valid.
381 if (!source_.get() || render_frame_id_ == -1)
382 return;
384 render_frame_id = render_frame_id_;
385 input_params = audio_processor_->InputFormat();
388 // Do nothing if the current buffer size is the WebRtc native buffer size.
389 if (GetBufferSize(input_params.sample_rate()) ==
390 input_params.frames_per_buffer()) {
391 return;
394 // Create a new audio stream as source which will open the hardware using
395 // WebRtc native buffer size.
396 SetCapturerSourceInternal(AudioDeviceFactory::NewInputDevice(render_frame_id),
397 input_params.channel_layout(),
398 input_params.sample_rate(),
402 void WebRtcAudioCapturer::Start() {
403 DCHECK(thread_checker_.CalledOnValidThread());
404 DVLOG(1) << "WebRtcAudioCapturer::Start()";
405 base::AutoLock auto_lock(lock_);
406 if (running_ || !source_.get())
407 return;
409 // Start the data source, i.e., start capturing data from the current source.
410 // We need to set the AGC control before starting the stream.
411 source_->SetAutomaticGainControl(true);
412 source_->Start();
413 running_ = true;
416 void WebRtcAudioCapturer::Stop() {
417 DCHECK(thread_checker_.CalledOnValidThread());
418 DVLOG(1) << "WebRtcAudioCapturer::Stop()";
419 scoped_refptr<media::AudioCapturerSource> source;
420 TrackList::ItemList tracks;
422 base::AutoLock auto_lock(lock_);
423 if (!running_)
424 return;
426 source = source_;
427 tracks = tracks_.Items();
428 tracks_.Clear();
429 running_ = false;
432 // Remove the capturer object from the WebRtcAudioDeviceImpl.
433 if (audio_device_)
434 audio_device_->RemoveAudioCapturer(this);
436 for (TrackList::ItemList::const_iterator it = tracks.begin();
437 it != tracks.end();
438 ++it) {
439 (*it)->Stop();
442 if (source.get())
443 source->Stop();
445 // Stop the audio processor to avoid feeding render data into the processor.
446 audio_processor_->Stop();
449 void WebRtcAudioCapturer::SetVolume(int volume) {
450 DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
451 DCHECK_LE(volume, MaxVolume());
452 double normalized_volume = static_cast<double>(volume) / MaxVolume();
453 base::AutoLock auto_lock(lock_);
454 if (source_.get())
455 source_->SetVolume(normalized_volume);
458 int WebRtcAudioCapturer::Volume() const {
459 base::AutoLock auto_lock(lock_);
460 return volume_;
463 int WebRtcAudioCapturer::MaxVolume() const {
464 return WebRtcAudioDeviceImpl::kMaxVolumeLevel;
467 media::AudioParameters WebRtcAudioCapturer::GetOutputFormat() const {
468 DCHECK(thread_checker_.CalledOnValidThread());
469 return audio_processor_->OutputFormat();
472 void WebRtcAudioCapturer::Capture(const media::AudioBus* audio_source,
473 int audio_delay_milliseconds,
474 double volume,
475 bool key_pressed) {
476 // This callback is driven by AudioInputDevice::AudioThreadCallback if
477 // |source_| is AudioInputDevice, otherwise it is driven by client's
478 // CaptureCallback.
479 #if defined(OS_WIN) || defined(OS_MACOSX)
480 DCHECK_LE(volume, 1.0);
481 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
482 // We have a special situation on Linux where the microphone volume can be
483 // "higher than maximum". The input volume slider in the sound preference
484 // allows the user to set a scaling that is higher than 100%. It means that
485 // even if the reported maximum levels is N, the actual microphone level can
486 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
487 DCHECK_LE(volume, 1.6);
488 #endif
490 // TODO(miu): Plumbing is needed to determine the actual capture timestamp
491 // of the audio, instead of just snapshotting TimeTicks::Now(), for proper
492 // audio/video sync. http://crbug.com/335335
493 const base::TimeTicks reference_clock_snapshot = base::TimeTicks::Now();
495 TrackList::ItemList tracks;
496 TrackList::ItemList tracks_to_notify_format;
497 int current_volume = 0;
499 base::AutoLock auto_lock(lock_);
500 if (!running_)
501 return;
503 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
504 // The volume can be higher than 255 on Linux, and it will be cropped to
505 // 255 since AGC does not allow values out of range.
506 volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
507 current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
508 tracks = tracks_.Items();
509 tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
512 DCHECK(audio_processor_->InputFormat().IsValid());
513 DCHECK_EQ(audio_source->channels(),
514 audio_processor_->InputFormat().channels());
515 DCHECK_EQ(audio_source->frames(),
516 audio_processor_->InputFormat().frames_per_buffer());
518 // Notify the tracks on when the format changes. This will do nothing if
519 // |tracks_to_notify_format| is empty.
520 const media::AudioParameters& output_params =
521 audio_processor_->OutputFormat();
522 for (const auto& track : tracks_to_notify_format) {
523 track->OnSetFormat(output_params);
524 track->SetAudioProcessor(audio_processor_);
527 // Figure out if the pre-processed data has any energy or not, the
528 // information will be passed to the track to force the calculator
529 // to report energy in case the post-processed data is zeroed by the audio
530 // processing.
531 const bool force_report_nonzero_energy = HasDataEnergy(*audio_source);
533 // Push the data to the processor for processing.
534 audio_processor_->PushCaptureData(
535 *audio_source,
536 base::TimeDelta::FromMilliseconds(audio_delay_milliseconds));
538 // Process and consume the data in the processor until there is not enough
539 // data in the processor.
540 media::AudioBus* processed_data = nullptr;
541 base::TimeDelta processed_data_audio_delay;
542 int new_volume = 0;
543 while (audio_processor_->ProcessAndConsumeData(
544 current_volume, key_pressed,
545 &processed_data, &processed_data_audio_delay, &new_volume)) {
546 DCHECK(processed_data);
547 const base::TimeTicks processed_data_capture_time =
548 reference_clock_snapshot - processed_data_audio_delay;
549 for (const auto& track : tracks) {
550 track->Capture(*processed_data,
551 processed_data_capture_time,
552 force_report_nonzero_energy);
555 if (new_volume) {
556 SetVolume(new_volume);
558 // Update the |current_volume| to avoid passing the old volume to AGC.
559 current_volume = new_volume;
564 void WebRtcAudioCapturer::OnCaptureError() {
565 NOTIMPLEMENTED();
568 media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const {
569 base::AutoLock auto_lock(lock_);
570 return audio_processor_.get() ? audio_processor_->InputFormat()
571 : media::AudioParameters();
574 bool WebRtcAudioCapturer::GetPairedOutputParameters(
575 int* session_id,
576 int* output_sample_rate,
577 int* output_frames_per_buffer) const {
578 // Don't set output parameters unless all of them are valid.
579 if (device_info_.session_id <= 0 ||
580 !device_info_.device.matched_output.sample_rate ||
581 !device_info_.device.matched_output.frames_per_buffer)
582 return false;
584 *session_id = device_info_.session_id;
585 *output_sample_rate = device_info_.device.matched_output.sample_rate;
586 *output_frames_per_buffer =
587 device_info_.device.matched_output.frames_per_buffer;
589 return true;
592 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
593 DCHECK(thread_checker_.CalledOnValidThread());
594 #if defined(OS_ANDROID)
595 // TODO(henrika): Tune and adjust buffer size on Android.
596 return (2 * sample_rate / 100);
597 #endif
599 // PeerConnection is running at a buffer size of 10ms data. A multiple of
600 // 10ms as the buffer size can give the best performance to PeerConnection.
601 int peer_connection_buffer_size = sample_rate / 100;
603 // Use the native hardware buffer size in non peer connection mode when the
604 // platform is using a native buffer size smaller than the PeerConnection
605 // buffer size and audio processing is off.
606 int hardware_buffer_size = device_info_.device.input.frames_per_buffer;
607 if (!peer_connection_mode_ && hardware_buffer_size &&
608 hardware_buffer_size <= peer_connection_buffer_size &&
609 !audio_processor_->has_audio_processing()) {
610 DVLOG(1) << "WebRtcAudioCapturer is using hardware buffer size "
611 << hardware_buffer_size;
612 return hardware_buffer_size;
615 return (sample_rate / 100);
618 void WebRtcAudioCapturer::SetCapturerSource(
619 const scoped_refptr<media::AudioCapturerSource>& source,
620 media::AudioParameters params) {
621 // Create a new audio stream as source which uses the new source.
622 SetCapturerSourceInternal(source,
623 params.channel_layout(),
624 params.sample_rate(),
628 } // namespace content