media/base/audio_converter.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // AudioConverter implementation.  Uses MultiChannelSincResampler for resampling
   6 // audio, ChannelMixer for channel mixing, and AudioPullFifo for buffering.
   7 //
   8 // Delay estimates are provided to InputCallbacks based on the frame delay
   9 // information reported via the resampler and FIFO units.
  10
  11 #include "media/base/audio_converter.h"
  12
  13 #include <algorithm>
  14
  15 #include "base/bind.h"
  16 #include "base/bind_helpers.h"
  17 #include "media/base/audio_bus.h"
  18 #include "media/base/audio_pull_fifo.h"
  19 #include "media/base/channel_mixer.h"
  20 #include "media/base/multi_channel_resampler.h"
  21 #include "media/base/vector_math.h"
  22
  23 namespace media {
  24
  25 AudioConverter::AudioConverter(const AudioParameters& input_params,
  26                                const AudioParameters& output_params,
  27                                bool disable_fifo)
  28     : chunk_size_(input_params.frames_per_buffer()),
  29       downmix_early_(false),
  30       resampler_frame_delay_(0),
  31       input_channel_count_(input_params.channels()) {
  32   CHECK(input_params.IsValid());
  33   CHECK(output_params.IsValid());
  34
  35   // Handle different input and output channel layouts.
  36   if (input_params.channel_layout() != output_params.channel_layout() ||
  37       input_params.channels() != output_params.channels()) {
  38     DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout()
  39              << " to " << output_params.channel_layout() << "; from "
  40              << input_params.channels() << " channels to "
  41              << output_params.channels() << " channels.";
  42     channel_mixer_.reset(new ChannelMixer(input_params, output_params));
  43
  44     // Pare off data as early as we can for efficiency.
  45     downmix_early_ = input_params.channels() > output_params.channels();
  46   }
  47
  48   // Only resample if necessary since it's expensive.
  49   if (input_params.sample_rate() != output_params.sample_rate()) {
  50     DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to "
  51              << output_params.sample_rate();
  52     const int request_size = disable_fifo ? SincResampler::kDefaultRequestSize :
  53         input_params.frames_per_buffer();
  54     const double io_sample_rate_ratio =
  55         input_params.sample_rate() /
  56         static_cast<double>(output_params.sample_rate());
  57     resampler_.reset(new MultiChannelResampler(
  58         downmix_early_ ? output_params.channels() : input_params.channels(),
  59         io_sample_rate_ratio,
  60         request_size,
  61         base::Bind(&AudioConverter::ProvideInput, base::Unretained(this))));
  62   }
  63
  64   input_frame_duration_ = base::TimeDelta::FromMicroseconds(
  65       base::Time::kMicrosecondsPerSecond /
  66       static_cast<double>(input_params.sample_rate()));
  67   output_frame_duration_ = base::TimeDelta::FromMicroseconds(
  68       base::Time::kMicrosecondsPerSecond /
  69       static_cast<double>(output_params.sample_rate()));
  70
  71   // The resampler can be configured to work with a specific request size, so a
  72   // FIFO is not necessary when resampling.
  73   if (disable_fifo || resampler_)
  74     return;
  75
  76   // Since the output device may want a different buffer size than the caller
  77   // asked for, we need to use a FIFO to ensure that both sides read in chunk
  78   // sizes they're configured for.
  79   if (input_params.frames_per_buffer() != output_params.frames_per_buffer()) {
  80     DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer()
  81              << " to " << output_params.frames_per_buffer();
  82     chunk_size_ = input_params.frames_per_buffer();
  83     audio_fifo_.reset(new AudioPullFifo(
  84         downmix_early_ ? output_params.channels() : input_params.channels(),
  85         chunk_size_,
  86         base::Bind(&AudioConverter::SourceCallback, base::Unretained(this))));
  87   }
  88 }
  89
  90 AudioConverter::~AudioConverter() {}
  91
  92 void AudioConverter::AddInput(InputCallback* input) {
  93   DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) ==
  94          transform_inputs_.end());
  95   transform_inputs_.push_back(input);
  96 }
  97
  98 void AudioConverter::RemoveInput(InputCallback* input) {
  99   DCHECK(std::find(transform_inputs_.begin(), transform_inputs_.end(), input) !=
 100          transform_inputs_.end());
 101   transform_inputs_.remove(input);
 102
 103   if (transform_inputs_.empty())
 104     Reset();
 105 }
 106
 107 void AudioConverter::Reset() {
 108   if (audio_fifo_)
 109     audio_fifo_->Clear();
 110   if (resampler_)
 111     resampler_->Flush();
 112 }
 113
 114 int AudioConverter::ChunkSize() const {
 115   if (!resampler_)
 116     return chunk_size_;
 117   return resampler_->ChunkSize();
 118 }
 119
 120 void AudioConverter::PrimeWithSilence() {
 121   if (resampler_) {
 122     resampler_->PrimeWithSilence();
 123   }
 124 }
 125
 126 void AudioConverter::ConvertWithDelay(const base::TimeDelta& initial_delay,
 127                                       AudioBus* dest) {
 128   initial_delay_ = initial_delay;
 129
 130   if (transform_inputs_.empty()) {
 131     dest->Zero();
 132     return;
 133   }
 134
 135   // Determine if channel mixing should be done and if it should be done before
 136   // or after resampling.  If it's possible to reduce the channel count prior to
 137   // resampling we can save a lot of processing time.  Vice versa, we don't want
 138   // to increase the channel count prior to resampling for the same reason.
 139   bool needs_mixing = channel_mixer_ && !downmix_early_;
 140
 141   if (needs_mixing)
 142     CreateUnmixedAudioIfNecessary(dest->frames());
 143
 144   AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest;
 145   DCHECK(temp_dest);
 146
 147   // Figure out which method to call based on whether we're resampling and
 148   // rebuffering, just resampling, or just mixing.  We want to avoid any extra
 149   // steps when possible since we may be converting audio data in real time.
 150   if (!resampler_ && !audio_fifo_) {
 151     SourceCallback(0, temp_dest);
 152   } else {
 153     if (resampler_)
 154       resampler_->Resample(temp_dest->frames(), temp_dest);
 155     else
 156       ProvideInput(0, temp_dest);
 157   }
 158
 159   // Finally upmix the channels if we didn't do so earlier.
 160   if (needs_mixing) {
 161     DCHECK_EQ(temp_dest->frames(), dest->frames());
 162     channel_mixer_->Transform(temp_dest, dest);
 163   }
 164 }
 165
 166 void AudioConverter::Convert(AudioBus* dest) {
 167   ConvertWithDelay(base::TimeDelta::FromMilliseconds(0), dest);
 168 }
 169
 170 void AudioConverter::SourceCallback(int fifo_frame_delay, AudioBus* dest) {
 171   const bool needs_downmix = channel_mixer_ && downmix_early_;
 172
 173   if (!mixer_input_audio_bus_ ||
 174       mixer_input_audio_bus_->frames() != dest->frames()) {
 175     mixer_input_audio_bus_ =
 176         AudioBus::Create(input_channel_count_, dest->frames());
 177   }
 178
 179   // If we're downmixing early we need a temporary AudioBus which matches
 180   // the the input channel count and input frame size since we're passing
 181   // |unmixed_audio_| directly to the |source_callback_|.
 182   if (needs_downmix)
 183     CreateUnmixedAudioIfNecessary(dest->frames());
 184
 185   AudioBus* const temp_dest = needs_downmix ? unmixed_audio_.get() : dest;
 186
 187   // Sanity check our inputs.
 188   DCHECK_EQ(temp_dest->frames(), mixer_input_audio_bus_->frames());
 189   DCHECK_EQ(temp_dest->channels(), mixer_input_audio_bus_->channels());
 190
 191   // Calculate the buffer delay for this callback.
 192   base::TimeDelta buffer_delay = initial_delay_;
 193   if (resampler_) {
 194     buffer_delay += base::TimeDelta::FromMicroseconds(
 195         resampler_frame_delay_ * output_frame_duration_.InMicroseconds());
 196   }
 197   if (audio_fifo_) {
 198     buffer_delay += base::TimeDelta::FromMicroseconds(
 199         fifo_frame_delay * input_frame_duration_.InMicroseconds());
 200   }
 201
 202   // If we only have a single input, avoid an extra copy.
 203   AudioBus* const provide_input_dest =
 204       transform_inputs_.size() == 1 ? temp_dest : mixer_input_audio_bus_.get();
 205
 206   // Have each mixer render its data into an output buffer then mix the result.
 207   for (auto* input : transform_inputs_) {
 208     const float volume = input->ProvideInput(provide_input_dest, buffer_delay);
 209
 210     // Optimize the most common single input, full volume case.
 211     if (input == transform_inputs_.front()) {
 212       if (volume == 1.0f) {
 213         if (temp_dest != provide_input_dest)
 214           provide_input_dest->CopyTo(temp_dest);
 215       } else if (volume > 0) {
 216         for (int i = 0; i < provide_input_dest->channels(); ++i) {
 217           vector_math::FMUL(
 218               provide_input_dest->channel(i), volume,
 219               provide_input_dest->frames(), temp_dest->channel(i));
 220         }
 221       } else {
 222         // Zero |temp_dest| otherwise, so we're mixing into a clean buffer.
 223         temp_dest->Zero();
 224       }
 225
 226       continue;
 227     }
 228
 229     // Volume adjust and mix each mixer input into |temp_dest| after rendering.
 230     if (volume > 0) {
 231       for (int i = 0; i < mixer_input_audio_bus_->channels(); ++i) {
 232         vector_math::FMAC(
 233             mixer_input_audio_bus_->channel(i), volume,
 234             mixer_input_audio_bus_->frames(), temp_dest->channel(i));
 235       }
 236     }
 237   }
 238
 239   if (needs_downmix) {
 240     DCHECK_EQ(temp_dest->frames(), dest->frames());
 241     channel_mixer_->Transform(temp_dest, dest);
 242   }
 243 }
 244
 245 void AudioConverter::ProvideInput(int resampler_frame_delay, AudioBus* dest) {
 246   resampler_frame_delay_ = resampler_frame_delay;
 247   if (audio_fifo_)
 248     audio_fifo_->Consume(dest, dest->frames());
 249   else
 250     SourceCallback(0, dest);
 251 }
 252
 253 void AudioConverter::CreateUnmixedAudioIfNecessary(int frames) {
 254   if (!unmixed_audio_ || unmixed_audio_->frames() != frames)
 255     unmixed_audio_ = AudioBus::Create(input_channel_count_, frames);
 256 }
 257
 258 }  // namespace media