cc: Make picture pile base thread safe.
[chromium-blink-merge.git] / content / renderer / media / webrtc_audio_capturer.cc
blob3299987d67b8eacb9a7eb84427a58d33ee147069
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/renderer/media/webrtc_audio_capturer.h"
7 #include "base/bind.h"
8 #include "base/logging.h"
9 #include "base/metrics/histogram.h"
10 #include "base/strings/string_util.h"
11 #include "base/strings/stringprintf.h"
12 #include "content/child/child_process.h"
13 #include "content/renderer/media/audio_device_factory.h"
14 #include "content/renderer/media/media_stream_audio_processor.h"
15 #include "content/renderer/media/media_stream_audio_processor_options.h"
16 #include "content/renderer/media/media_stream_audio_source.h"
17 #include "content/renderer/media/webrtc_audio_device_impl.h"
18 #include "content/renderer/media/webrtc_local_audio_track.h"
19 #include "content/renderer/media/webrtc_logging.h"
20 #include "media/audio/sample_rates.h"
22 namespace content {
24 namespace {
26 // Method to check if any of the data in |audio_source| has energy.
27 bool HasDataEnergy(const media::AudioBus& audio_source) {
28 for (int ch = 0; ch < audio_source.channels(); ++ch) {
29 const float* channel_ptr = audio_source.channel(ch);
30 for (int frame = 0; frame < audio_source.frames(); ++frame) {
31 if (channel_ptr[frame] != 0)
32 return true;
36 // All the data is zero.
37 return false;
40 } // namespace
42 // Reference counted container of WebRtcLocalAudioTrack delegate.
43 // TODO(xians): Switch to MediaStreamAudioSinkOwner.
44 class WebRtcAudioCapturer::TrackOwner
45 : public base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner> {
46 public:
47 explicit TrackOwner(WebRtcLocalAudioTrack* track)
48 : delegate_(track) {}
50 void Capture(const int16* audio_data,
51 base::TimeDelta delay,
52 double volume,
53 bool key_pressed,
54 bool need_audio_processing,
55 bool force_report_nonzero_energy) {
56 base::AutoLock lock(lock_);
57 if (delegate_) {
58 delegate_->Capture(audio_data,
59 delay,
60 volume,
61 key_pressed,
62 need_audio_processing,
63 force_report_nonzero_energy);
67 void OnSetFormat(const media::AudioParameters& params) {
68 base::AutoLock lock(lock_);
69 if (delegate_)
70 delegate_->OnSetFormat(params);
73 void SetAudioProcessor(
74 const scoped_refptr<MediaStreamAudioProcessor>& processor) {
75 base::AutoLock lock(lock_);
76 if (delegate_)
77 delegate_->SetAudioProcessor(processor);
80 void Reset() {
81 base::AutoLock lock(lock_);
82 delegate_ = NULL;
85 void Stop() {
86 base::AutoLock lock(lock_);
87 DCHECK(delegate_);
89 // This can be reentrant so reset |delegate_| before calling out.
90 WebRtcLocalAudioTrack* temp = delegate_;
91 delegate_ = NULL;
92 temp->Stop();
95 // Wrapper which allows to use std::find_if() when adding and removing
96 // sinks to/from the list.
97 struct TrackWrapper {
98 TrackWrapper(WebRtcLocalAudioTrack* track) : track_(track) {}
99 bool operator()(
100 const scoped_refptr<WebRtcAudioCapturer::TrackOwner>& owner) const {
101 return owner->IsEqual(track_);
103 WebRtcLocalAudioTrack* track_;
106 protected:
107 virtual ~TrackOwner() {}
109 private:
110 friend class base::RefCountedThreadSafe<WebRtcAudioCapturer::TrackOwner>;
112 bool IsEqual(const WebRtcLocalAudioTrack* other) const {
113 base::AutoLock lock(lock_);
114 return (other == delegate_);
117 // Do NOT reference count the |delegate_| to avoid cyclic reference counting.
118 WebRtcLocalAudioTrack* delegate_;
119 mutable base::Lock lock_;
121 DISALLOW_COPY_AND_ASSIGN(TrackOwner);
124 // static
125 scoped_refptr<WebRtcAudioCapturer> WebRtcAudioCapturer::CreateCapturer(
126 int render_view_id, const StreamDeviceInfo& device_info,
127 const blink::WebMediaConstraints& constraints,
128 WebRtcAudioDeviceImpl* audio_device,
129 MediaStreamAudioSource* audio_source) {
130 scoped_refptr<WebRtcAudioCapturer> capturer = new WebRtcAudioCapturer(
131 render_view_id, device_info, constraints, audio_device, audio_source);
132 if (capturer->Initialize())
133 return capturer;
135 return NULL;
138 bool WebRtcAudioCapturer::Initialize() {
139 DCHECK(thread_checker_.CalledOnValidThread());
140 DVLOG(1) << "WebRtcAudioCapturer::Initialize()";
141 WebRtcLogMessage(base::StringPrintf(
142 "WAC::Initialize. render_view_id=%d"
143 ", channel_layout=%d, sample_rate=%d, buffer_size=%d"
144 ", session_id=%d, paired_output_sample_rate=%d"
145 ", paired_output_frames_per_buffer=%d, effects=%d. ",
146 render_view_id_,
147 device_info_.device.input.channel_layout,
148 device_info_.device.input.sample_rate,
149 device_info_.device.input.frames_per_buffer,
150 device_info_.session_id,
151 device_info_.device.matched_output.sample_rate,
152 device_info_.device.matched_output.frames_per_buffer,
153 device_info_.device.input.effects));
155 if (render_view_id_ == -1) {
156 // Return true here to allow injecting a new source via
157 // SetCapturerSourceForTesting() at a later state.
158 return true;
161 MediaAudioConstraints audio_constraints(constraints_,
162 device_info_.device.input.effects);
163 if (!audio_constraints.IsValid())
164 return false;
166 media::ChannelLayout channel_layout = static_cast<media::ChannelLayout>(
167 device_info_.device.input.channel_layout);
169 // If KEYBOARD_MIC effect is set, change the layout to the corresponding
170 // layout that includes the keyboard mic.
171 if ((device_info_.device.input.effects &
172 media::AudioParameters::KEYBOARD_MIC) &&
173 MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() &&
174 audio_constraints.GetProperty(
175 MediaAudioConstraints::kGoogExperimentalNoiseSuppression)) {
176 if (channel_layout == media::CHANNEL_LAYOUT_STEREO) {
177 channel_layout = media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC;
178 DVLOG(1) << "Changed stereo layout to stereo + keyboard mic layout due "
179 << "to KEYBOARD_MIC effect.";
180 } else {
181 DVLOG(1) << "KEYBOARD_MIC effect ignored, not compatible with layout "
182 << channel_layout;
186 DVLOG(1) << "Audio input hardware channel layout: " << channel_layout;
187 UMA_HISTOGRAM_ENUMERATION("WebRTC.AudioInputChannelLayout",
188 channel_layout, media::CHANNEL_LAYOUT_MAX + 1);
190 // Verify that the reported input channel configuration is supported.
191 if (channel_layout != media::CHANNEL_LAYOUT_MONO &&
192 channel_layout != media::CHANNEL_LAYOUT_STEREO &&
193 channel_layout != media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) {
194 DLOG(ERROR) << channel_layout
195 << " is not a supported input channel configuration.";
196 return false;
199 DVLOG(1) << "Audio input hardware sample rate: "
200 << device_info_.device.input.sample_rate;
201 media::AudioSampleRate asr;
202 if (media::ToAudioSampleRate(device_info_.device.input.sample_rate, &asr)) {
203 UMA_HISTOGRAM_ENUMERATION(
204 "WebRTC.AudioInputSampleRate", asr, media::kAudioSampleRateMax + 1);
205 } else {
206 UMA_HISTOGRAM_COUNTS("WebRTC.AudioInputSampleRateUnexpected",
207 device_info_.device.input.sample_rate);
210 // Create and configure the default audio capturing source.
211 SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id_),
212 channel_layout,
213 static_cast<float>(device_info_.device.input.sample_rate));
215 // Add the capturer to the WebRtcAudioDeviceImpl since it needs some hardware
216 // information from the capturer.
217 if (audio_device_)
218 audio_device_->AddAudioCapturer(this);
220 return true;
223 WebRtcAudioCapturer::WebRtcAudioCapturer(
224 int render_view_id,
225 const StreamDeviceInfo& device_info,
226 const blink::WebMediaConstraints& constraints,
227 WebRtcAudioDeviceImpl* audio_device,
228 MediaStreamAudioSource* audio_source)
229 : constraints_(constraints),
230 audio_processor_(new rtc::RefCountedObject<MediaStreamAudioProcessor>(
231 constraints,
232 device_info.device.input.effects,
233 audio_device)),
234 running_(false),
235 render_view_id_(render_view_id),
236 device_info_(device_info),
237 volume_(0),
238 peer_connection_mode_(false),
239 key_pressed_(false),
240 need_audio_processing_(false),
241 audio_device_(audio_device),
242 audio_source_(audio_source) {
243 DVLOG(1) << "WebRtcAudioCapturer::WebRtcAudioCapturer()";
246 WebRtcAudioCapturer::~WebRtcAudioCapturer() {
247 DCHECK(thread_checker_.CalledOnValidThread());
248 DCHECK(tracks_.IsEmpty());
249 DVLOG(1) << "WebRtcAudioCapturer::~WebRtcAudioCapturer()";
250 Stop();
253 void WebRtcAudioCapturer::AddTrack(WebRtcLocalAudioTrack* track) {
254 DCHECK(track);
255 DVLOG(1) << "WebRtcAudioCapturer::AddTrack()";
258 base::AutoLock auto_lock(lock_);
259 // Verify that |track| is not already added to the list.
260 DCHECK(!tracks_.Contains(TrackOwner::TrackWrapper(track)));
262 // Add with a tag, so we remember to call OnSetFormat() on the new
263 // track.
264 scoped_refptr<TrackOwner> track_owner(new TrackOwner(track));
265 tracks_.AddAndTag(track_owner.get());
269 void WebRtcAudioCapturer::RemoveTrack(WebRtcLocalAudioTrack* track) {
270 DCHECK(thread_checker_.CalledOnValidThread());
271 DVLOG(1) << "WebRtcAudioCapturer::RemoveTrack()";
272 bool stop_source = false;
274 base::AutoLock auto_lock(lock_);
276 scoped_refptr<TrackOwner> removed_item =
277 tracks_.Remove(TrackOwner::TrackWrapper(track));
279 // Clear the delegate to ensure that no more capture callbacks will
280 // be sent to this sink. Also avoids a possible crash which can happen
281 // if this method is called while capturing is active.
282 if (removed_item.get()) {
283 removed_item->Reset();
284 stop_source = tracks_.IsEmpty();
287 if (stop_source) {
288 // Since WebRtcAudioCapturer does not inherit MediaStreamAudioSource,
289 // and instead MediaStreamAudioSource is composed of a WebRtcAudioCapturer,
290 // we have to call StopSource on the MediaStreamSource. This will call
291 // MediaStreamAudioSource::DoStopSource which in turn call
292 // WebRtcAudioCapturerer::Stop();
293 audio_source_->StopSource();
297 void WebRtcAudioCapturer::SetCapturerSource(
298 const scoped_refptr<media::AudioCapturerSource>& source,
299 media::ChannelLayout channel_layout,
300 float sample_rate) {
301 DCHECK(thread_checker_.CalledOnValidThread());
302 DVLOG(1) << "SetCapturerSource(channel_layout=" << channel_layout << ","
303 << "sample_rate=" << sample_rate << ")";
304 scoped_refptr<media::AudioCapturerSource> old_source;
306 base::AutoLock auto_lock(lock_);
307 if (source_.get() == source.get())
308 return;
310 source_.swap(old_source);
311 source_ = source;
313 // Reset the flag to allow starting the new source.
314 running_ = false;
317 DVLOG(1) << "Switching to a new capture source.";
318 if (old_source.get())
319 old_source->Stop();
321 // Dispatch the new parameters both to the sink(s) and to the new source,
322 // also apply the new |constraints|.
323 // The idea is to get rid of any dependency of the microphone parameters
324 // which would normally be used by default.
325 // bits_per_sample is always 16 for now.
326 int buffer_size = GetBufferSize(sample_rate);
327 media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
328 channel_layout, sample_rate,
329 16, buffer_size,
330 device_info_.device.input.effects);
333 base::AutoLock auto_lock(lock_);
334 // Notify the |audio_processor_| of the new format.
335 audio_processor_->OnCaptureFormatChanged(params);
337 MediaAudioConstraints audio_constraints(constraints_,
338 device_info_.device.input.effects);
339 need_audio_processing_ = audio_constraints.NeedsAudioProcessing();
340 // Notify all tracks about the new format.
341 tracks_.TagAll();
344 if (source.get())
345 source->Initialize(params, this, session_id());
347 Start();
350 void WebRtcAudioCapturer::EnablePeerConnectionMode() {
351 DCHECK(thread_checker_.CalledOnValidThread());
352 DVLOG(1) << "EnablePeerConnectionMode";
353 // Do nothing if the peer connection mode has been enabled.
354 if (peer_connection_mode_)
355 return;
357 peer_connection_mode_ = true;
358 int render_view_id = -1;
359 media::AudioParameters input_params;
361 base::AutoLock auto_lock(lock_);
362 // Simply return if there is no existing source or the |render_view_id_| is
363 // not valid.
364 if (!source_.get() || render_view_id_== -1)
365 return;
367 render_view_id = render_view_id_;
368 input_params = audio_processor_->InputFormat();
371 // Do nothing if the current buffer size is the WebRtc native buffer size.
372 if (GetBufferSize(input_params.sample_rate()) ==
373 input_params.frames_per_buffer()) {
374 return;
377 // Create a new audio stream as source which will open the hardware using
378 // WebRtc native buffer size.
379 SetCapturerSource(AudioDeviceFactory::NewInputDevice(render_view_id),
380 input_params.channel_layout(),
381 static_cast<float>(input_params.sample_rate()));
384 void WebRtcAudioCapturer::Start() {
385 DCHECK(thread_checker_.CalledOnValidThread());
386 DVLOG(1) << "WebRtcAudioCapturer::Start()";
387 base::AutoLock auto_lock(lock_);
388 if (running_ || !source_.get())
389 return;
391 // Start the data source, i.e., start capturing data from the current source.
392 // We need to set the AGC control before starting the stream.
393 source_->SetAutomaticGainControl(true);
394 source_->Start();
395 running_ = true;
398 void WebRtcAudioCapturer::Stop() {
399 DCHECK(thread_checker_.CalledOnValidThread());
400 DVLOG(1) << "WebRtcAudioCapturer::Stop()";
401 scoped_refptr<media::AudioCapturerSource> source;
402 TrackList::ItemList tracks;
404 base::AutoLock auto_lock(lock_);
405 if (!running_)
406 return;
408 source = source_;
409 tracks = tracks_.Items();
410 tracks_.Clear();
411 running_ = false;
414 // Remove the capturer object from the WebRtcAudioDeviceImpl.
415 if (audio_device_)
416 audio_device_->RemoveAudioCapturer(this);
418 for (TrackList::ItemList::const_iterator it = tracks.begin();
419 it != tracks.end();
420 ++it) {
421 (*it)->Stop();
424 if (source.get())
425 source->Stop();
427 // Stop the audio processor to avoid feeding render data into the processor.
428 audio_processor_->Stop();
431 void WebRtcAudioCapturer::SetVolume(int volume) {
432 DVLOG(1) << "WebRtcAudioCapturer::SetVolume()";
433 DCHECK_LE(volume, MaxVolume());
434 double normalized_volume = static_cast<double>(volume) / MaxVolume();
435 base::AutoLock auto_lock(lock_);
436 if (source_.get())
437 source_->SetVolume(normalized_volume);
440 int WebRtcAudioCapturer::Volume() const {
441 base::AutoLock auto_lock(lock_);
442 return volume_;
445 int WebRtcAudioCapturer::MaxVolume() const {
446 return WebRtcAudioDeviceImpl::kMaxVolumeLevel;
449 void WebRtcAudioCapturer::Capture(const media::AudioBus* audio_source,
450 int audio_delay_milliseconds,
451 double volume,
452 bool key_pressed) {
453 // This callback is driven by AudioInputDevice::AudioThreadCallback if
454 // |source_| is AudioInputDevice, otherwise it is driven by client's
455 // CaptureCallback.
456 #if defined(OS_WIN) || defined(OS_MACOSX)
457 DCHECK_LE(volume, 1.0);
458 #elif (defined(OS_LINUX) && !defined(OS_CHROMEOS)) || defined(OS_OPENBSD)
459 // We have a special situation on Linux where the microphone volume can be
460 // "higher than maximum". The input volume slider in the sound preference
461 // allows the user to set a scaling that is higher than 100%. It means that
462 // even if the reported maximum levels is N, the actual microphone level can
463 // go up to 1.5x*N and that corresponds to a normalized |volume| of 1.5x.
464 DCHECK_LE(volume, 1.6);
465 #endif
467 TrackList::ItemList tracks;
468 TrackList::ItemList tracks_to_notify_format;
469 int current_volume = 0;
470 base::TimeDelta audio_delay;
471 bool need_audio_processing = true;
473 base::AutoLock auto_lock(lock_);
474 if (!running_)
475 return;
477 // Map internal volume range of [0.0, 1.0] into [0, 255] used by AGC.
478 // The volume can be higher than 255 on Linux, and it will be cropped to
479 // 255 since AGC does not allow values out of range.
480 volume_ = static_cast<int>((volume * MaxVolume()) + 0.5);
481 current_volume = volume_ > MaxVolume() ? MaxVolume() : volume_;
482 audio_delay = base::TimeDelta::FromMilliseconds(audio_delay_milliseconds);
483 audio_delay_ = audio_delay;
484 key_pressed_ = key_pressed;
485 tracks = tracks_.Items();
486 tracks_.RetrieveAndClearTags(&tracks_to_notify_format);
488 // Set the flag to turn on the audio processing in PeerConnection level.
489 // Note that, we turn off the audio processing in PeerConnection if the
490 // processor has already processed the data.
491 need_audio_processing = need_audio_processing_ ?
492 !MediaStreamAudioProcessor::IsAudioTrackProcessingEnabled() : false;
495 DCHECK(audio_processor_->InputFormat().IsValid());
496 DCHECK_EQ(audio_source->channels(),
497 audio_processor_->InputFormat().channels());
498 DCHECK_EQ(audio_source->frames(),
499 audio_processor_->InputFormat().frames_per_buffer());
501 // Notify the tracks on when the format changes. This will do nothing if
502 // |tracks_to_notify_format| is empty.
503 media::AudioParameters output_params = audio_processor_->OutputFormat();
504 for (TrackList::ItemList::const_iterator it = tracks_to_notify_format.begin();
505 it != tracks_to_notify_format.end(); ++it) {
506 (*it)->OnSetFormat(output_params);
507 (*it)->SetAudioProcessor(audio_processor_);
510 // Figure out if the pre-processed data has any energy or not, the
511 // information will be passed to the track to force the calculator
512 // to report energy in case the post-processed data is zeroed by the audio
513 // processing.
514 const bool force_report_nonzero_energy = HasDataEnergy(*audio_source);
516 // Push the data to the processor for processing.
517 audio_processor_->PushCaptureData(audio_source);
519 // Process and consume the data in the processor until there is not enough
520 // data in the processor.
521 int16* output = NULL;
522 int new_volume = 0;
523 while (audio_processor_->ProcessAndConsumeData(
524 audio_delay, current_volume, key_pressed, &new_volume, &output)) {
525 // Feed the post-processed data to the tracks.
526 for (TrackList::ItemList::const_iterator it = tracks.begin();
527 it != tracks.end(); ++it) {
528 (*it)->Capture(output, audio_delay, current_volume, key_pressed,
529 need_audio_processing, force_report_nonzero_energy);
532 if (new_volume) {
533 SetVolume(new_volume);
535 // Update the |current_volume| to avoid passing the old volume to AGC.
536 current_volume = new_volume;
541 void WebRtcAudioCapturer::OnCaptureError() {
542 NOTIMPLEMENTED();
545 media::AudioParameters WebRtcAudioCapturer::source_audio_parameters() const {
546 base::AutoLock auto_lock(lock_);
547 return audio_processor_.get() ? audio_processor_->InputFormat()
548 : media::AudioParameters();
551 bool WebRtcAudioCapturer::GetPairedOutputParameters(
552 int* session_id,
553 int* output_sample_rate,
554 int* output_frames_per_buffer) const {
555 // Don't set output parameters unless all of them are valid.
556 if (device_info_.session_id <= 0 ||
557 !device_info_.device.matched_output.sample_rate ||
558 !device_info_.device.matched_output.frames_per_buffer)
559 return false;
561 *session_id = device_info_.session_id;
562 *output_sample_rate = device_info_.device.matched_output.sample_rate;
563 *output_frames_per_buffer =
564 device_info_.device.matched_output.frames_per_buffer;
566 return true;
569 int WebRtcAudioCapturer::GetBufferSize(int sample_rate) const {
570 DCHECK(thread_checker_.CalledOnValidThread());
571 #if defined(OS_ANDROID)
572 // TODO(henrika): Tune and adjust buffer size on Android.
573 return (2 * sample_rate / 100);
574 #endif
576 // PeerConnection is running at a buffer size of 10ms data. A multiple of
577 // 10ms as the buffer size can give the best performance to PeerConnection.
578 int peer_connection_buffer_size = sample_rate / 100;
580 // Use the native hardware buffer size in non peer connection mode when the
581 // platform is using a native buffer size smaller than the PeerConnection
582 // buffer size.
583 int hardware_buffer_size = device_info_.device.input.frames_per_buffer;
584 if (!peer_connection_mode_ && hardware_buffer_size &&
585 hardware_buffer_size <= peer_connection_buffer_size) {
586 return hardware_buffer_size;
589 return (sample_rate / 100);
592 void WebRtcAudioCapturer::GetAudioProcessingParams(
593 base::TimeDelta* delay, int* volume, bool* key_pressed) {
594 base::AutoLock auto_lock(lock_);
595 *delay = audio_delay_;
596 *volume = volume_;
597 *key_pressed = key_pressed_;
600 void WebRtcAudioCapturer::SetCapturerSourceForTesting(
601 const scoped_refptr<media::AudioCapturerSource>& source,
602 media::AudioParameters params) {
603 // Create a new audio stream as source which uses the new source.
604 SetCapturerSource(source, params.channel_layout(),
605 static_cast<float>(params.sample_rate()));
608 } // namespace content