1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
6 #define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
8 #include "base/atomicops.h"
9 #include "base/files/file.h"
10 #include "base/synchronization/lock.h"
11 #include "base/threading/thread_checker.h"
12 #include "base/time/time.h"
13 #include "content/common/content_export.h"
14 #include "content/renderer/media/aec_dump_message_filter.h"
15 #include "content/renderer/media/webrtc_audio_device_impl.h"
16 #include "media/base/audio_converter.h"
17 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
18 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
19 #include "third_party/webrtc/modules/interface/module_common_types.h"
22 class WebMediaConstraints
;
28 class AudioParameters
;
33 class TypingDetection
;
38 class EchoInformation
;
39 class MediaStreamAudioBus
;
40 class MediaStreamAudioFifo
;
41 class RTCMediaConstraints
;
43 using webrtc::AudioProcessorInterface
;
45 // This class owns an object of webrtc::AudioProcessing which contains signal
46 // processing components like AGC, AEC and NS. It enables the components based
47 // on the getUserMedia constraints, processes the data and outputs it in a unit
48 // of 10 ms data chunk.
49 class CONTENT_EXPORT MediaStreamAudioProcessor
:
50 NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink
),
51 NON_EXPORTED_BASE(public AudioProcessorInterface
),
52 NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate
) {
54 // |playout_data_source| is used to register this class as a sink to the
55 // WebRtc playout data for processing AEC. If clients do not enable AEC,
56 // |playout_data_source| won't be used.
57 MediaStreamAudioProcessor(const blink::WebMediaConstraints
& constraints
,
59 WebRtcPlayoutDataSource
* playout_data_source
);
61 // Called when the format of the capture data has changed.
62 // Called on the main render thread. The caller is responsible for stopping
63 // the capture thread before calling this method.
64 // After this method, the capture thread will be changed to a new capture
66 void OnCaptureFormatChanged(const media::AudioParameters
& source_params
);
68 // Pushes capture data in |audio_source| to the internal FIFO. Each call to
69 // this method should be followed by calls to ProcessAndConsumeData() while
70 // it returns false, to pull out all available data.
71 // Called on the capture audio thread.
72 void PushCaptureData(const media::AudioBus
& audio_source
,
73 base::TimeDelta capture_delay
);
75 // Processes a block of 10 ms data from the internal FIFO, returning true if
76 // |processed_data| contains the result. Returns false and does not modify the
77 // outputs if the internal FIFO has insufficient data. The caller does NOT own
78 // the object pointed to by |*processed_data|.
79 // |capture_delay| is an adjustment on the |capture_delay| value provided in
80 // the last call to PushCaptureData().
81 // |new_volume| receives the new microphone volume from the AGC.
82 // The new microphone volume range is [0, 255], and the value will be 0 if
83 // the microphone volume should not be adjusted.
84 // Called on the capture audio thread.
85 bool ProcessAndConsumeData(
88 media::AudioBus
** processed_data
,
89 base::TimeDelta
* capture_delay
,
92 // Stops the audio processor, no more AEC dump or render data after calling
96 // The audio formats of the capture input to and output from the processor.
97 // Must only be called on the main render or audio capture threads.
98 const media::AudioParameters
& InputFormat() const;
99 const media::AudioParameters
& OutputFormat() const;
101 // Accessor to check if the audio processing is enabled or not.
102 bool has_audio_processing() const { return audio_processing_
!= NULL
; }
104 // AecDumpMessageFilter::AecDumpDelegate implementation.
105 // Called on the main render thread.
106 void OnAecDumpFile(const IPC::PlatformFileForTransit
& file_handle
) override
;
107 void OnDisableAecDump() override
;
108 void OnIpcClosing() override
;
111 ~MediaStreamAudioProcessor() override
;
114 friend class MediaStreamAudioProcessorTest
;
115 FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest
,
116 GetAecDumpMessageFilter
);
118 // WebRtcPlayoutDataSource::Sink implementation.
119 void OnPlayoutData(media::AudioBus
* audio_bus
,
121 int audio_delay_milliseconds
) override
;
122 void OnPlayoutDataSourceChanged() override
;
124 // webrtc::AudioProcessorInterface implementation.
125 // This method is called on the libjingle thread.
126 void GetStats(AudioProcessorStats
* stats
) override
;
128 // Helper to initialize the WebRtc AudioProcessing.
129 void InitializeAudioProcessingModule(
130 const blink::WebMediaConstraints
& constraints
, int effects
);
131 void ConfigureBeamforming(webrtc::Config
* config
);
133 // Helper to initialize the capture converter.
134 void InitializeCaptureFifo(const media::AudioParameters
& input_format
);
136 // Helper to initialize the render converter.
137 void InitializeRenderFifoIfNeeded(int sample_rate
,
138 int number_of_channels
,
139 int frames_per_buffer
);
141 // Called by ProcessAndConsumeData().
142 // Returns the new microphone volume in the range of |0, 255].
143 // When the volume does not need to be updated, it returns 0.
144 int ProcessData(const float* const* process_ptrs
,
146 base::TimeDelta capture_delay
,
149 float* const* output_ptrs
);
151 // Cached value for the render delay latency. This member is accessed by
152 // both the capture audio thread and the render audio thread.
153 base::subtle::Atomic32 render_delay_ms_
;
155 // Module to handle processing and format conversion.
156 scoped_ptr
<webrtc::AudioProcessing
> audio_processing_
;
158 // FIFO to provide 10 ms capture chunks.
159 scoped_ptr
<MediaStreamAudioFifo
> capture_fifo_
;
160 // Receives processing output.
161 scoped_ptr
<MediaStreamAudioBus
> output_bus_
;
163 // FIFO to provide 10 ms render chunks when the AEC is enabled.
164 scoped_ptr
<MediaStreamAudioFifo
> render_fifo_
;
166 // These are mutated on the main render thread in OnCaptureFormatChanged().
167 // The caller guarantees this does not run concurrently with accesses on the
168 // capture audio thread.
169 media::AudioParameters input_format_
;
170 media::AudioParameters output_format_
;
171 // Only used on the render audio thread.
172 media::AudioParameters render_format_
;
174 // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
175 // lifetime of RenderThread.
176 WebRtcPlayoutDataSource
* playout_data_source_
;
178 // Used to DCHECK that some methods are called on the main render thread.
179 base::ThreadChecker main_thread_checker_
;
180 // Used to DCHECK that some methods are called on the capture audio thread.
181 base::ThreadChecker capture_thread_checker_
;
182 // Used to DCHECK that some methods are called on the render audio thread.
183 base::ThreadChecker render_thread_checker_
;
185 // Flag to enable stereo channel mirroring.
186 bool audio_mirroring_
;
188 scoped_ptr
<webrtc::TypingDetection
> typing_detector_
;
189 // This flag is used to show the result of typing detection.
190 // It can be accessed by the capture audio thread and by the libjingle thread
191 // which calls GetStats().
192 base::subtle::Atomic32 typing_detected_
;
194 // Communication with browser for AEC dump.
195 scoped_refptr
<AecDumpMessageFilter
> aec_dump_message_filter_
;
197 // Flag to avoid executing Stop() more than once.
200 // Object for logging echo information when the AEC is enabled. Accessible by
201 // the libjingle thread through GetStats().
202 scoped_ptr
<EchoInformation
> echo_information_
;
204 // Flag is enabled if AudioProcessing supports 48kHz sample rate.
205 bool audio_proc_48kHz_support_
;
207 DISALLOW_COPY_AND_ASSIGN(MediaStreamAudioProcessor
);
210 } // namespace content
212 #endif // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_