content/renderer/media/media_stream_audio_processor.h

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
   6 #define CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_
   7
   8 #include "base/atomicops.h"
   9 #include "base/files/file.h"
  10 #include "base/synchronization/lock.h"
  11 #include "base/threading/thread_checker.h"
  12 #include "base/time/time.h"
  13 #include "content/common/content_export.h"
  14 #include "content/renderer/media/aec_dump_message_filter.h"
  15 #include "content/renderer/media/webrtc_audio_device_impl.h"
  16 #include "media/base/audio_converter.h"
  17 #include "third_party/libjingle/source/talk/app/webrtc/mediastreaminterface.h"
  18 #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
  19
  20 namespace blink {
  21 class WebMediaConstraints;
  22 }
  23
  24 namespace media {
  25 class AudioBus;
  26 class AudioFifo;
  27 class AudioParameters;
  28 }  // namespace media
  29
  30 namespace webrtc {
  31 class TypingDetection;
  32 }
  33
  34 namespace content {
  35
  36 class EchoInformation;
  37 class MediaStreamAudioBus;
  38 class MediaStreamAudioFifo;
  39 class RTCMediaConstraints;
  40
  41 using webrtc::AudioProcessorInterface;
  42
  43 // This class owns an object of webrtc::AudioProcessing which contains signal
  44 // processing components like AGC, AEC and NS. It enables the components based
  45 // on the getUserMedia constraints, processes the data and outputs it in a unit
  46 // of 10 ms data chunk.
  47 class CONTENT_EXPORT MediaStreamAudioProcessor :
  48     NON_EXPORTED_BASE(public WebRtcPlayoutDataSource::Sink),
  49     NON_EXPORTED_BASE(public AudioProcessorInterface),
  50     NON_EXPORTED_BASE(public AecDumpMessageFilter::AecDumpDelegate) {
  51  public:
  52   // |playout_data_source| is used to register this class as a sink to the
  53   // WebRtc playout data for processing AEC. If clients do not enable AEC,
  54   // |playout_data_source| won't be used.
  55   MediaStreamAudioProcessor(const blink::WebMediaConstraints& constraints,
  56                             int effects,
  57                             WebRtcPlayoutDataSource* playout_data_source);
  58
  59   // Called when the format of the capture data has changed.
  60   // Called on the main render thread. The caller is responsible for stopping
  61   // the capture thread before calling this method.
  62   // After this method, the capture thread will be changed to a new capture
  63   // thread.
  64   void OnCaptureFormatChanged(const media::AudioParameters& source_params);
  65
  66   // Pushes capture data in |audio_source| to the internal FIFO. Each call to
  67   // this method should be followed by calls to ProcessAndConsumeData() while
  68   // it returns false, to pull out all available data.
  69   // Called on the capture audio thread.
  70   void PushCaptureData(const media::AudioBus& audio_source,
  71                        base::TimeDelta capture_delay);
  72
  73   // Processes a block of 10 ms data from the internal FIFO, returning true if
  74   // |processed_data| contains the result. Returns false and does not modify the
  75   // outputs if the internal FIFO has insufficient data. The caller does NOT own
  76   // the object pointed to by |*processed_data|.
  77   // |capture_delay| is an adjustment on the |capture_delay| value provided in
  78   // the last call to PushCaptureData().
  79   // |new_volume| receives the new microphone volume from the AGC.
  80   // The new microphone volume range is [0, 255], and the value will be 0 if
  81   // the microphone volume should not be adjusted.
  82   // Called on the capture audio thread.
  83   bool ProcessAndConsumeData(
  84       int volume,
  85       bool key_pressed,
  86       media::AudioBus** processed_data,
  87       base::TimeDelta* capture_delay,
  88       int* new_volume);
  89
  90   // Stops the audio processor, no more AEC dump or render data after calling
  91   // this method.
  92   void Stop();
  93
  94   // The audio formats of the capture input to and output from the processor.
  95   // Must only be called on the main render or audio capture threads.
  96   const media::AudioParameters& InputFormat() const;
  97   const media::AudioParameters& OutputFormat() const;
  98
  99   // Accessor to check if the audio processing is enabled or not.
 100   bool has_audio_processing() const { return audio_processing_ != NULL; }
 101
 102   // AecDumpMessageFilter::AecDumpDelegate implementation.
 103   // Called on the main render thread.
 104   void OnAecDumpFile(const IPC::PlatformFileForTransit& file_handle) override;
 105   void OnDisableAecDump() override;
 106   void OnIpcClosing() override;
 107
 108  protected:
 109   ~MediaStreamAudioProcessor() override;
 110
 111  private:
 112   friend class MediaStreamAudioProcessorTest;
 113   FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
 114                            GetAecDumpMessageFilter);
 115
 116   // WebRtcPlayoutDataSource::Sink implementation.
 117   void OnPlayoutData(media::AudioBus* audio_bus,
 118                      int sample_rate,
 119                      int audio_delay_milliseconds) override;
 120   void OnPlayoutDataSourceChanged() override;
 121
 122   // webrtc::AudioProcessorInterface implementation.
 123   // This method is called on the libjingle thread.
 124   void GetStats(AudioProcessorStats* stats) override;
 125
 126   // Helper to initialize the WebRtc AudioProcessing.
 127   void InitializeAudioProcessingModule(
 128       const blink::WebMediaConstraints& constraints, int effects);
 129   void ConfigureBeamforming(webrtc::Config* config,
 130                             const std::string& geometry_str) const;
 131
 132   // Parses the array geometry from the URL string formatted as
 133   // "x1 y1 z1 ... xn yn zn" for an n-microphone array.
 134   // Returns a zero-sized vector if |geometry_str| isn't a parseable geometry.
 135   std::vector<webrtc::Point> ParseArrayGeometry(
 136       const std::string& geometry_str) const;
 137
 138   // Helper to initialize the capture converter.
 139   void InitializeCaptureFifo(const media::AudioParameters& input_format);
 140
 141   // Helper to initialize the render converter.
 142   void InitializeRenderFifoIfNeeded(int sample_rate,
 143                                     int number_of_channels,
 144                                     int frames_per_buffer);
 145
 146   // Called by ProcessAndConsumeData().
 147   // Returns the new microphone volume in the range of |0, 255].
 148   // When the volume does not need to be updated, it returns 0.
 149   int ProcessData(const float* const* process_ptrs,
 150                   int process_frames,
 151                   base::TimeDelta capture_delay,
 152                   int volume,
 153                   bool key_pressed,
 154                   float* const* output_ptrs);
 155
 156   // Cached value for the render delay latency. This member is accessed by
 157   // both the capture audio thread and the render audio thread.
 158   base::subtle::Atomic32 render_delay_ms_;
 159
 160   // Module to handle processing and format conversion.
 161   scoped_ptr<webrtc::AudioProcessing> audio_processing_;
 162
 163   // FIFO to provide 10 ms capture chunks.
 164   scoped_ptr<MediaStreamAudioFifo> capture_fifo_;
 165   // Receives processing output.
 166   scoped_ptr<MediaStreamAudioBus> output_bus_;
 167
 168   // FIFO to provide 10 ms render chunks when the AEC is enabled.
 169   scoped_ptr<MediaStreamAudioFifo> render_fifo_;
 170
 171   // These are mutated on the main render thread in OnCaptureFormatChanged().
 172   // The caller guarantees this does not run concurrently with accesses on the
 173   // capture audio thread.
 174   media::AudioParameters input_format_;
 175   media::AudioParameters output_format_;
 176   // Only used on the render audio thread.
 177   media::AudioParameters render_format_;
 178
 179   // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
 180   // lifetime of RenderThread.
 181   WebRtcPlayoutDataSource* playout_data_source_;
 182
 183   // Used to DCHECK that some methods are called on the main render thread.
 184   base::ThreadChecker main_thread_checker_;
 185   // Used to DCHECK that some methods are called on the capture audio thread.
 186   base::ThreadChecker capture_thread_checker_;
 187   // Used to DCHECK that some methods are called on the render audio thread.
 188   base::ThreadChecker render_thread_checker_;
 189
 190   // Flag to enable stereo channel mirroring.
 191   bool audio_mirroring_;
 192
 193   scoped_ptr<webrtc::TypingDetection> typing_detector_;
 194   // This flag is used to show the result of typing detection.
 195   // It can be accessed by the capture audio thread and by the libjingle thread
 196   // which calls GetStats().
 197   base::subtle::Atomic32 typing_detected_;
 198
 199   // Communication with browser for AEC dump.
 200   scoped_refptr<AecDumpMessageFilter> aec_dump_message_filter_;
 201
 202   // Flag to avoid executing Stop() more than once.
 203   bool stopped_;
 204
 205   // Object for logging echo information when the AEC is enabled. Accessible by
 206   // the libjingle thread through GetStats().
 207   scoped_ptr<EchoInformation> echo_information_;
 208
 209   DISALLOW_COPY_AND_ASSIGN(MediaStreamAudioProcessor);
 210 };
 211
 212 }  // namespace content
 213
 214 #endif  // CONTENT_RENDERER_MEDIA_MEDIA_STREAM_AUDIO_PROCESSOR_H_