dom/media/driftcontrol/DynamicResampler.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "DynamicResampler.h"
   7
   8 namespace mozilla {
   9
  10 DynamicResampler::DynamicResampler(uint32_t aInRate, uint32_t aOutRate,
  11                                    uint32_t aInputPreBufferFrameCount)
  12     : mOutRate(aOutRate),
  13       mInputPreBufferFrameCount(aInputPreBufferFrameCount),
  14       mInRate(aInRate) {
  15   MOZ_ASSERT(aInRate);
  16   MOZ_ASSERT(aOutRate);
  17   UpdateResampler(mInRate, STEREO);
  18   mInputStreamFile.Open("DynamicResamplerInFirstChannel", 1, mInRate);
  19   mOutputStreamFile.Open("DynamicResamplerOutFirstChannel", 1, mOutRate);
  20 }
  21
  22 DynamicResampler::~DynamicResampler() {
  23   if (mResampler) {
  24     speex_resampler_destroy(mResampler);
  25   }
  26 }
  27
  28 void DynamicResampler::SetSampleFormat(AudioSampleFormat aFormat) {
  29   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_SILENCE);
  30   MOZ_ASSERT(aFormat == AUDIO_FORMAT_S16 || aFormat == AUDIO_FORMAT_FLOAT32);
  31
  32   mSampleFormat = aFormat;
  33   for (AudioRingBuffer& b : mInternalInBuffer) {
  34     b.SetSampleFormat(mSampleFormat);
  35   }
  36
  37   // Pre-allocate something big.
  38   // EnsureInputBufferDuration() adds 50ms for jitter to this first allocation
  39   // so the 50ms argument means at least 100ms.
  40   EnsureInputBufferSizeInFrames(mInRate / 20);
  41 }
  42
  43 void DynamicResampler::EnsurePreBuffer(media::TimeUnit aDuration) {
  44   if (mIsPreBufferSet) {
  45     return;
  46   }
  47
  48   uint32_t buffered = mInternalInBuffer[0].AvailableRead();
  49   if (buffered == 0) {
  50     // Wait for the first input segment before deciding how much to pre-buffer.
  51     // If it is large it indicates high-latency, and the buffer would have to
  52     // handle that.  This also means that the pre-buffer is not set up just
  53     // before a large input segment would extend the buffering beyond the
  54     // desired level.
  55     return;
  56   }
  57
  58   mIsPreBufferSet = true;
  59
  60   uint32_t needed =
  61       aDuration.ToTicksAtRate(mInRate) + mInputPreBufferFrameCount;
  62   EnsureInputBufferSizeInFrames(needed);
  63
  64   if (needed > buffered) {
  65     for (auto& b : mInternalInBuffer) {
  66       b.PrependSilence(needed - buffered);
  67     }
  68   } else if (needed < buffered) {
  69     for (auto& b : mInternalInBuffer) {
  70       b.Discard(buffered - needed);
  71     }
  72   }
  73 }
  74
  75 void DynamicResampler::SetInputPreBufferFrameCount(
  76     uint32_t aInputPreBufferFrameCount) {
  77   mInputPreBufferFrameCount = aInputPreBufferFrameCount;
  78 }
  79
  80 bool DynamicResampler::Resample(float* aOutBuffer, uint32_t aOutFrames,
  81                                 uint32_t aChannelIndex) {
  82   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32);
  83   return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex);
  84 }
  85
  86 bool DynamicResampler::Resample(int16_t* aOutBuffer, uint32_t aOutFrames,
  87                                 uint32_t aChannelIndex) {
  88   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16);
  89   return ResampleInternal(aOutBuffer, aOutFrames, aChannelIndex);
  90 }
  91
  92 void DynamicResampler::ResampleInternal(const float* aInBuffer,
  93                                         uint32_t* aInFrames, float* aOutBuffer,
  94                                         uint32_t* aOutFrames,
  95                                         uint32_t aChannelIndex) {
  96   MOZ_ASSERT(mResampler);
  97   MOZ_ASSERT(mChannels);
  98   MOZ_ASSERT(mInRate);
  99   MOZ_ASSERT(mOutRate);
 100
 101   MOZ_ASSERT(aInFrames);
 102   MOZ_ASSERT(*aInFrames > 0);
 103   MOZ_ASSERT(aOutBuffer);
 104   MOZ_ASSERT(aOutFrames);
 105   MOZ_ASSERT(*aOutFrames > 0);
 106
 107   MOZ_ASSERT(aChannelIndex <= mChannels);
 108
 109 #ifdef DEBUG
 110   int rv =
 111 #endif
 112       speex_resampler_process_float(mResampler, aChannelIndex, aInBuffer,
 113                                     aInFrames, aOutBuffer, aOutFrames);
 114   MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS);
 115
 116   if (aChannelIndex == 0 && !mIsWarmingUp) {
 117     mInputStreamFile.Write(aInBuffer, *aInFrames);
 118     mOutputStreamFile.Write(aOutBuffer, *aOutFrames);
 119   }
 120 }
 121
 122 void DynamicResampler::ResampleInternal(const int16_t* aInBuffer,
 123                                         uint32_t* aInFrames,
 124                                         int16_t* aOutBuffer,
 125                                         uint32_t* aOutFrames,
 126                                         uint32_t aChannelIndex) {
 127   MOZ_ASSERT(mResampler);
 128   MOZ_ASSERT(mChannels);
 129   MOZ_ASSERT(mInRate);
 130   MOZ_ASSERT(mOutRate);
 131
 132   MOZ_ASSERT(aInFrames);
 133   MOZ_ASSERT(*aInFrames > 0);
 134   MOZ_ASSERT(aOutBuffer);
 135   MOZ_ASSERT(aOutFrames);
 136   MOZ_ASSERT(*aOutFrames > 0);
 137
 138   MOZ_ASSERT(aChannelIndex <= mChannels);
 139
 140 #ifdef DEBUG
 141   int rv =
 142 #endif
 143       speex_resampler_process_int(mResampler, aChannelIndex, aInBuffer,
 144                                   aInFrames, aOutBuffer, aOutFrames);
 145   MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS);
 146
 147   if (aChannelIndex == 0 && !mIsWarmingUp) {
 148     mInputStreamFile.Write(aInBuffer, *aInFrames);
 149     mOutputStreamFile.Write(aOutBuffer, *aOutFrames);
 150   }
 151 }
 152
 153 void DynamicResampler::UpdateResampler(uint32_t aInRate, uint32_t aChannels) {
 154   MOZ_ASSERT(aInRate);
 155   MOZ_ASSERT(aChannels);
 156
 157   if (mChannels != aChannels) {
 158     uint32_t bufferSizeInFrames = InFramesBufferSize();
 159     if (mResampler) {
 160       speex_resampler_destroy(mResampler);
 161     }
 162     mResampler = speex_resampler_init(aChannels, aInRate, mOutRate,
 163                                       SPEEX_RESAMPLER_QUALITY_MIN, nullptr);
 164     MOZ_ASSERT(mResampler);
 165     mChannels = aChannels;
 166     mInRate = aInRate;
 167     mResamplerIsBypassed &= aInRate == mOutRate;
 168     // Between mono and stereo changes, keep always allocated 2 channels to
 169     // avoid reallocations in the most common case.
 170     if ((mChannels == STEREO || mChannels == 1) &&
 171         mInternalInBuffer.Length() == STEREO) {
 172       // Don't worry if format is not set it will write silence then.
 173       if ((mSampleFormat == AUDIO_FORMAT_S16 ||
 174            mSampleFormat == AUDIO_FORMAT_FLOAT32) &&
 175           mChannels == STEREO) {
 176         // The mono channel is always up to date. When we are going from mono
 177         // to stereo upmix the mono to stereo channel
 178         uint32_t bufferedDuration = mInternalInBuffer[0].AvailableRead();
 179         mInternalInBuffer[1].Clear();
 180         if (bufferedDuration) {
 181           mInternalInBuffer[1].Write(mInternalInBuffer[0], bufferedDuration);
 182         }
 183       }
 184       // Maintain stereo size
 185       mInputTail.SetLength(STEREO);
 186       WarmUpResampler(false);
 187       return;
 188     }
 189     // upmix or downmix, for now just clear but it has to be updated
 190     // because allocates and this is executed in audio thread.
 191     mInternalInBuffer.Clear();
 192     for (uint32_t i = 0; i < mChannels; ++i) {
 193       AudioRingBuffer* b = mInternalInBuffer.AppendElement(0);
 194
 195       if (mSampleFormat != AUDIO_FORMAT_SILENCE) {
 196         // In ctor this update is not needed
 197         b->SetSampleFormat(mSampleFormat);
 198       }
 199     }
 200     EnsureInputBufferSizeInFrames(bufferSizeInFrames);
 201     mInputTail.SetLength(mChannels);
 202     return;
 203   }
 204
 205   if (mInRate != aInRate) {
 206     // If the rates was the same the resampler was not being used so warm up.
 207     if (mResamplerIsBypassed) {
 208       mResamplerIsBypassed = false;
 209       WarmUpResampler(true);
 210     }
 211
 212 #ifdef DEBUG
 213     int rv =
 214 #endif
 215         speex_resampler_set_rate(mResampler, aInRate, mOutRate);
 216     MOZ_ASSERT(rv == RESAMPLER_ERR_SUCCESS);
 217     mInRate = aInRate;
 218   }
 219 }
 220
 221 void DynamicResampler::WarmUpResampler(bool aSkipLatency) {
 222   MOZ_ASSERT(mInputTail.Length());
 223   mIsWarmingUp = true;
 224   for (uint32_t i = 0; i < mChannels; ++i) {
 225     if (!mInputTail[i].Length()) {
 226       continue;
 227     }
 228     uint32_t inFrames = mInputTail[i].Length();
 229     uint32_t outFrames = 5 * TailBuffer::MAXSIZE;  // something big
 230     if (mSampleFormat == AUDIO_FORMAT_S16) {
 231       short outBuffer[5 * TailBuffer::MAXSIZE] = {};
 232       ResampleInternal(mInputTail[i].Buffer<short>(), &inFrames, outBuffer,
 233                        &outFrames, i);
 234       MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length());
 235     } else {
 236       float outBuffer[100] = {};
 237       ResampleInternal(mInputTail[i].Buffer<float>(), &inFrames, outBuffer,
 238                        &outFrames, i);
 239       MOZ_ASSERT(inFrames == (uint32_t)mInputTail[i].Length());
 240     }
 241   }
 242   if (aSkipLatency) {
 243     // Don't generate output frames corresponding to times before the next
 244     // input sample.
 245     speex_resampler_skip_zeros(mResampler);
 246   }
 247   mIsWarmingUp = false;
 248 }
 249
 250 void DynamicResampler::AppendInput(Span<const float* const> aInBuffer,
 251                                    uint32_t aInFrames) {
 252   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_FLOAT32);
 253   AppendInputInternal(aInBuffer, aInFrames);
 254 }
 255 void DynamicResampler::AppendInput(Span<const int16_t* const> aInBuffer,
 256                                    uint32_t aInFrames) {
 257   MOZ_ASSERT(mSampleFormat == AUDIO_FORMAT_S16);
 258   AppendInputInternal(aInBuffer, aInFrames);
 259 }
 260
 261 void DynamicResampler::AppendInputSilence(const uint32_t aInFrames) {
 262   MOZ_ASSERT(aInFrames);
 263   MOZ_ASSERT(mChannels);
 264   MOZ_ASSERT(mInternalInBuffer.Length() >= (uint32_t)mChannels);
 265   for (uint32_t i = 0; i < mChannels; ++i) {
 266     mInternalInBuffer[i].WriteSilence(aInFrames);
 267   }
 268 }
 269
 270 uint32_t DynamicResampler::InFramesBufferSize() const {
 271   if (mSampleFormat == AUDIO_FORMAT_SILENCE) {
 272     return 0;
 273   }
 274   // Buffers may have different capacities if a memory allocation has failed.
 275   MOZ_ASSERT(!mInternalInBuffer.IsEmpty());
 276   uint32_t min = std::numeric_limits<uint32_t>::max();
 277   for (const auto& b : mInternalInBuffer) {
 278     min = std::min(min, b.Capacity());
 279   }
 280   return min;
 281 }
 282
 283 uint32_t DynamicResampler::InFramesBuffered(uint32_t aChannelIndex) const {
 284   MOZ_ASSERT(mChannels);
 285   MOZ_ASSERT(aChannelIndex <= mChannels);
 286   MOZ_ASSERT(aChannelIndex <= mInternalInBuffer.Length());
 287   if (!mIsPreBufferSet) {
 288     return mInputPreBufferFrameCount;
 289   }
 290   return mInternalInBuffer[aChannelIndex].AvailableRead();
 291 }
 292
 293 }  // namespace mozilla