alc/voice.cpp

   1 /**
   2  * OpenAL cross platform audio library
   3  * Copyright (C) 1999-2007 by authors.
   4  * This library is free software; you can redistribute it and/or
   5  *  modify it under the terms of the GNU Library General Public
   6  *  License as published by the Free Software Foundation; either
   7  *  version 2 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  *  Library General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Library General Public
  15  *  License along with this library; if not, write to the
  16  *  Free Software Foundation, Inc.,
  17  *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18  * Or go to http://www.gnu.org/copyleft/lgpl.html
  19  */
  20
  21 #include "config.h"
  22
  23 #include "voice.h"
  24
  25 #include <algorithm>
  26 #include <array>
  27 #include <atomic>
  28 #include <cassert>
  29 #include <climits>
  30 #include <cstddef>
  31 #include <cstdint>
  32 #include <iterator>
  33 #include <memory>
  34 #include <new>
  35 #include <utility>
  36
  37 #include "AL/al.h"
  38 #include "AL/alc.h"
  39
  40 #include "al/buffer.h"
  41 #include "al/event.h"
  42 #include "al/source.h"
  43 #include "alcmain.h"
  44 #include "albyte.h"
  45 #include "alconfig.h"
  46 #include "alcontext.h"
  47 #include "alnumeric.h"
  48 #include "aloptional.h"
  49 #include "alspan.h"
  50 #include "alstring.h"
  51 #include "alu.h"
  52 #include "cpu_caps.h"
  53 #include "devformat.h"
  54 #include "filters/biquad.h"
  55 #include "filters/nfc.h"
  56 #include "filters/splitter.h"
  57 #include "fmt_traits.h"
  58 #include "hrtf.h"
  59 #include "inprogext.h"
  60 #include "logging.h"
  61 #include "mixer/defs.h"
  62 #include "opthelpers.h"
  63 #include "ringbuffer.h"
  64 #include "threads.h"
  65 #include "vector.h"
  66
  67 struct CTag;
  68 #ifdef HAVE_SSE
  69 struct SSETag;
  70 #endif
  71 #ifdef HAVE_NEON
  72 struct NEONTag;
  73 #endif
  74 struct CopyTag;
  75
  76
  77 static_assert((BUFFERSIZE-1)/MAX_PITCH > 0, "MAX_PITCH is too large for BUFFERSIZE!");
  78 static_assert((INT_MAX>>FRACTIONBITS)/MAX_PITCH > BUFFERSIZE,
  79     "MAX_PITCH and/or BUFFERSIZE are too large for FRACTIONBITS!");
  80
  81
  82 Resampler ResamplerDefault{Resampler::Linear};
  83
  84 MixerFunc MixSamples{Mix_<CTag>};
  85
  86 namespace {
  87
  88 using HrtfMixerFunc = void(*)(const float *InSamples, float2 *AccumSamples, const ALuint IrSize,
  89     const MixHrtfFilter *hrtfparams, const size_t BufferSize);
  90 using HrtfMixerBlendFunc = void(*)(const float *InSamples, float2 *AccumSamples,
  91     const ALuint IrSize, const HrtfFilter *oldparams, const MixHrtfFilter *newparams,
  92     const size_t BufferSize);
  93
  94 HrtfMixerFunc MixHrtfSamples{MixHrtf_<CTag>};
  95 HrtfMixerBlendFunc MixHrtfBlendSamples{MixHrtfBlend_<CTag>};
  96
  97 inline MixerFunc SelectMixer()
  98 {
  99 #ifdef HAVE_NEON
 100     if((CPUCapFlags&CPU_CAP_NEON))
 101         return Mix_<NEONTag>;
 102 #endif
 103 #ifdef HAVE_SSE
 104     if((CPUCapFlags&CPU_CAP_SSE))
 105         return Mix_<SSETag>;
 106 #endif
 107     return Mix_<CTag>;
 108 }
 109
 110 inline HrtfMixerFunc SelectHrtfMixer()
 111 {
 112 #ifdef HAVE_NEON
 113     if((CPUCapFlags&CPU_CAP_NEON))
 114         return MixHrtf_<NEONTag>;
 115 #endif
 116 #ifdef HAVE_SSE
 117     if((CPUCapFlags&CPU_CAP_SSE))
 118         return MixHrtf_<SSETag>;
 119 #endif
 120     return MixHrtf_<CTag>;
 121 }
 122
 123 inline HrtfMixerBlendFunc SelectHrtfBlendMixer()
 124 {
 125 #ifdef HAVE_NEON
 126     if((CPUCapFlags&CPU_CAP_NEON))
 127         return MixHrtfBlend_<NEONTag>;
 128 #endif
 129 #ifdef HAVE_SSE
 130     if((CPUCapFlags&CPU_CAP_SSE))
 131         return MixHrtfBlend_<SSETag>;
 132 #endif
 133     return MixHrtfBlend_<CTag>;
 134 }
 135
 136 } // namespace
 137
 138
 139 void aluInitMixer()
 140 {
 141     if(auto resopt = ConfigValueStr(nullptr, nullptr, "resampler"))
 142     {
 143         struct ResamplerEntry {
 144             const char name[16];
 145             const Resampler resampler;
 146         };
 147         constexpr ResamplerEntry ResamplerList[]{
 148             { "none", Resampler::Point },
 149             { "point", Resampler::Point },
 150             { "linear", Resampler::Linear },
 151             { "cubic", Resampler::Cubic },
 152             { "bsinc12", Resampler::BSinc12 },
 153             { "fast_bsinc12", Resampler::FastBSinc12 },
 154             { "bsinc24", Resampler::BSinc24 },
 155             { "fast_bsinc24", Resampler::FastBSinc24 },
 156         };
 157
 158         const char *str{resopt->c_str()};
 159         if(al::strcasecmp(str, "bsinc") == 0)
 160         {
 161             WARN("Resampler option \"%s\" is deprecated, using bsinc12\n", str);
 162             str = "bsinc12";
 163         }
 164         else if(al::strcasecmp(str, "sinc4") == 0 || al::strcasecmp(str, "sinc8") == 0)
 165         {
 166             WARN("Resampler option \"%s\" is deprecated, using cubic\n", str);
 167             str = "cubic";
 168         }
 169
 170         auto iter = std::find_if(std::begin(ResamplerList), std::end(ResamplerList),
 171             [str](const ResamplerEntry &entry) -> bool
 172             { return al::strcasecmp(str, entry.name) == 0; });
 173         if(iter == std::end(ResamplerList))
 174             ERR("Invalid resampler: %s\n", str);
 175         else
 176             ResamplerDefault = iter->resampler;
 177     }
 178
 179     MixSamples = SelectMixer();
 180     MixHrtfBlendSamples = SelectHrtfBlendMixer();
 181     MixHrtfSamples = SelectHrtfMixer();
 182 }
 183
 184
 185 namespace {
 186
 187 void SendSourceStoppedEvent(ALCcontext *context, ALuint id)
 188 {
 189     RingBuffer *ring{context->mAsyncEvents.get()};
 190     auto evt_vec = ring->getWriteVector();
 191     if(evt_vec.first.len < 1) return;
 192
 193     AsyncEvent *evt{::new(evt_vec.first.buf) AsyncEvent{EventType_SourceStateChange}};
 194     evt->u.srcstate.id = id;
 195     evt->u.srcstate.state = AL_STOPPED;
 196
 197     ring->writeAdvance(1);
 198 }
 199
 200
 201 const float *DoFilters(BiquadFilter &lpfilter, BiquadFilter &hpfilter, float *dst,
 202     const al::span<const float> src, int type)
 203 {
 204     switch(type)
 205     {
 206     case AF_None:
 207         lpfilter.clear();
 208         hpfilter.clear();
 209         break;
 210
 211     case AF_LowPass:
 212         lpfilter.process(src, dst);
 213         hpfilter.clear();
 214         return dst;
 215     case AF_HighPass:
 216         lpfilter.clear();
 217         hpfilter.process(src, dst);
 218         return dst;
 219
 220     case AF_BandPass:
 221         DualBiquad{lpfilter, hpfilter}.process(src, dst);
 222         return dst;
 223     }
 224     return src.data();
 225 }
 226
 227
 228 void LoadSamples(float *RESTRICT dst, const al::byte *src, const size_t srcstep, FmtType srctype,
 229     const size_t samples) noexcept
 230 {
 231 #define HANDLE_FMT(T)  case T: al::LoadSampleArray<T>(dst, src, srcstep, samples); break
 232     switch(srctype)
 233     {
 234     HANDLE_FMT(FmtUByte);
 235     HANDLE_FMT(FmtShort);
 236     HANDLE_FMT(FmtFloat);
 237     HANDLE_FMT(FmtDouble);
 238     HANDLE_FMT(FmtMulaw);
 239     HANDLE_FMT(FmtAlaw);
 240     }
 241 #undef HANDLE_FMT
 242 }
 243
 244 float *LoadBufferStatic(ALbufferlistitem *BufferListItem, ALbufferlistitem *&BufferLoopItem,
 245     const size_t NumChannels, const size_t SampleSize, const size_t chan, size_t DataPosInt,
 246     al::span<float> SrcBuffer)
 247 {
 248     const ALbuffer *Buffer{BufferListItem->mBuffer};
 249     const ALuint LoopStart{Buffer->LoopStart};
 250     const ALuint LoopEnd{Buffer->LoopEnd};
 251     ASSUME(LoopEnd > LoopStart);
 252
 253     /* If current pos is beyond the loop range, do not loop */
 254     if(!BufferLoopItem || DataPosInt >= LoopEnd)
 255     {
 256         BufferLoopItem = nullptr;
 257
 258         /* Load what's left to play from the buffer */
 259         const size_t DataRem{minz(SrcBuffer.size(), Buffer->mBuffer.mSampleLen-DataPosInt)};
 260
 261         const al::byte *Data{Buffer->mBuffer.mData.data()};
 262         Data += (DataPosInt*NumChannels + chan)*SampleSize;
 263
 264         LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataRem);
 265         SrcBuffer = SrcBuffer.subspan(DataRem);
 266     }
 267     else
 268     {
 269         /* Load what's left of this loop iteration */
 270         const size_t DataRem{minz(SrcBuffer.size(), LoopEnd-DataPosInt)};
 271
 272         const al::byte *Data{Buffer->mBuffer.mData.data()};
 273         Data += (DataPosInt*NumChannels + chan)*SampleSize;
 274
 275         LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataRem);
 276         SrcBuffer = SrcBuffer.subspan(DataRem);
 277
 278         /* Load any repeats of the loop we can to fill the buffer. */
 279         const auto LoopSize = static_cast<size_t>(LoopEnd - LoopStart);
 280         while(!SrcBuffer.empty())
 281         {
 282             const size_t DataSize{minz(SrcBuffer.size(), LoopSize)};
 283
 284             Data = Buffer->mBuffer.mData.data() + (LoopStart*NumChannels + chan)*SampleSize;
 285
 286             LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataSize);
 287             SrcBuffer = SrcBuffer.subspan(DataSize);
 288         }
 289     }
 290     return SrcBuffer.begin();
 291 }
 292
 293 float *LoadBufferCallback(ALbufferlistitem *BufferListItem, const size_t NumChannels,
 294     const size_t SampleSize, const size_t chan, size_t NumCallbackSamples,
 295     al::span<float> SrcBuffer)
 296 {
 297     const ALbuffer *Buffer{BufferListItem->mBuffer};
 298
 299     /* Load what's left to play from the buffer */
 300     const size_t DataRem{minz(SrcBuffer.size(), NumCallbackSamples)};
 301
 302     const al::byte *Data{Buffer->mBuffer.mData.data() + chan*SampleSize};
 303
 304     LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataRem);
 305     SrcBuffer = SrcBuffer.subspan(DataRem);
 306
 307     return SrcBuffer.begin();
 308 }
 309
 310 float *LoadBufferQueue(ALbufferlistitem *BufferListItem, ALbufferlistitem *BufferLoopItem,
 311     const size_t NumChannels, const size_t SampleSize, const size_t chan, size_t DataPosInt,
 312     al::span<float> SrcBuffer)
 313 {
 314     /* Crawl the buffer queue to fill in the temp buffer */
 315     while(BufferListItem && !SrcBuffer.empty())
 316     {
 317         ALbuffer *Buffer{BufferListItem->mBuffer};
 318         if(!(Buffer && DataPosInt < Buffer->mBuffer.mSampleLen))
 319         {
 320             if(Buffer) DataPosInt -= Buffer->mBuffer.mSampleLen;
 321             BufferListItem = BufferListItem->mNext.load(std::memory_order_acquire);
 322             if(!BufferListItem) BufferListItem = BufferLoopItem;
 323             continue;
 324         }
 325
 326         const size_t DataSize{minz(SrcBuffer.size(), Buffer->mBuffer.mSampleLen-DataPosInt)};
 327
 328         const al::byte *Data{Buffer->mBuffer.mData.data()};
 329         Data += (DataPosInt*NumChannels + chan)*SampleSize;
 330
 331         LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataSize);
 332         SrcBuffer = SrcBuffer.subspan(DataSize);
 333         if(SrcBuffer.empty()) break;
 334
 335         DataPosInt = 0;
 336         BufferListItem = BufferListItem->mNext.load(std::memory_order_acquire);
 337         if(!BufferListItem) BufferListItem = BufferLoopItem;
 338     }
 339
 340     return SrcBuffer.begin();
 341 }
 342
 343
 344 void DoHrtfMix(const float *samples, const ALuint DstBufferSize, DirectParams &parms,
 345     const float TargetGain, const ALuint Counter, ALuint OutPos, const ALuint IrSize,
 346     ALCdevice *Device)
 347 {
 348     auto &HrtfSamples = Device->HrtfSourceData;
 349     /* Source HRTF mixing needs to include the direct delay so it remains
 350      * aligned with the direct mix's HRTF filtering.
 351      */
 352     float2 *AccumSamples{Device->HrtfAccumData + HRTF_DIRECT_DELAY};
 353
 354     /* Copy the HRTF history and new input samples into a temp buffer. */
 355     auto src_iter = std::copy(parms.Hrtf.History.begin(), parms.Hrtf.History.end(),
 356         std::begin(HrtfSamples));
 357     std::copy_n(samples, DstBufferSize, src_iter);
 358     /* Copy the last used samples back into the history buffer for later. */
 359     std::copy_n(std::begin(HrtfSamples) + DstBufferSize, parms.Hrtf.History.size(),
 360         parms.Hrtf.History.begin());
 361
 362     /* If fading and this is the first mixing pass, fade between the IRs. */
 363     ALuint fademix{0u};
 364     if(Counter && OutPos == 0)
 365     {
 366         fademix = minu(DstBufferSize, Counter);
 367
 368         float gain{TargetGain};
 369
 370         /* The new coefficients need to fade in completely since they're
 371          * replacing the old ones. To keep the gain fading consistent,
 372          * interpolate between the old and new target gains given how much of
 373          * the fade time this mix handles.
 374          */
 375         if(Counter > fademix)
 376         {
 377             const float a{static_cast<float>(fademix) / static_cast<float>(Counter)};
 378             gain = lerp(parms.Hrtf.Old.Gain, TargetGain, a);
 379         }
 380         MixHrtfFilter hrtfparams;
 381         hrtfparams.Coeffs = &parms.Hrtf.Target.Coeffs;
 382         hrtfparams.Delay = parms.Hrtf.Target.Delay;
 383         hrtfparams.Gain = 0.0f;
 384         hrtfparams.GainStep = gain / static_cast<float>(fademix);
 385
 386         MixHrtfBlendSamples(HrtfSamples, AccumSamples+OutPos, IrSize, &parms.Hrtf.Old, &hrtfparams,
 387             fademix);
 388         /* Update the old parameters with the result. */
 389         parms.Hrtf.Old = parms.Hrtf.Target;
 390         parms.Hrtf.Old.Gain = gain;
 391         OutPos += fademix;
 392     }
 393
 394     if(fademix < DstBufferSize)
 395     {
 396         const ALuint todo{DstBufferSize - fademix};
 397         float gain{TargetGain};
 398
 399         /* Interpolate the target gain if the gain fading lasts longer than
 400          * this mix.
 401          */
 402         if(Counter > DstBufferSize)
 403         {
 404             const float a{static_cast<float>(todo) / static_cast<float>(Counter-fademix)};
 405             gain = lerp(parms.Hrtf.Old.Gain, TargetGain, a);
 406         }
 407
 408         MixHrtfFilter hrtfparams;
 409         hrtfparams.Coeffs = &parms.Hrtf.Target.Coeffs;
 410         hrtfparams.Delay = parms.Hrtf.Target.Delay;
 411         hrtfparams.Gain = parms.Hrtf.Old.Gain;
 412         hrtfparams.GainStep = (gain - parms.Hrtf.Old.Gain) / static_cast<float>(todo);
 413         MixHrtfSamples(HrtfSamples+fademix, AccumSamples+OutPos, IrSize, &hrtfparams, todo);
 414         /* Store the now-current gain for next time. */
 415         parms.Hrtf.Old.Gain = gain;
 416     }
 417 }
 418
 419 void DoNfcMix(const al::span<const float> samples, FloatBufferLine *OutBuffer, DirectParams &parms,
 420     const float *TargetGains, const ALuint Counter, const ALuint OutPos, ALCdevice *Device)
 421 {
 422     using FilterProc = void (NfcFilter::*)(const al::span<const float>, float*);
 423     static constexpr FilterProc NfcProcess[MAX_AMBI_ORDER+1]{
 424         nullptr, &NfcFilter::process1, &NfcFilter::process2, &NfcFilter::process3};
 425
 426     float *CurrentGains{parms.Gains.Current.data()};
 427     MixSamples(samples, {OutBuffer, 1u}, CurrentGains, TargetGains, Counter, OutPos);
 428     ++OutBuffer;
 429     ++CurrentGains;
 430     ++TargetGains;
 431
 432     const al::span<float> nfcsamples{Device->NfcSampleData, samples.size()};
 433     size_t order{1};
 434     while(const size_t chancount{Device->NumChannelsPerOrder[order]})
 435     {
 436         (parms.NFCtrlFilter.*NfcProcess[order])(samples, nfcsamples.data());
 437         MixSamples(nfcsamples, {OutBuffer, chancount}, CurrentGains, TargetGains, Counter, OutPos);
 438         OutBuffer += chancount;
 439         CurrentGains += chancount;
 440         TargetGains += chancount;
 441         if(++order == MAX_AMBI_ORDER+1)
 442             break;
 443     }
 444 }
 445
 446 } // namespace
 447
 448 void Voice::mix(const State vstate, ALCcontext *Context, const ALuint SamplesToDo)
 449 {
 450     static constexpr std::array<float,MAX_OUTPUT_CHANNELS> SilentTarget{};
 451
 452     ASSUME(SamplesToDo > 0);
 453
 454     /* Get voice info */
 455     ALuint DataPosInt{mPosition.load(std::memory_order_relaxed)};
 456     ALuint DataPosFrac{mPositionFrac.load(std::memory_order_relaxed)};
 457     ALbufferlistitem *BufferListItem{mCurrentBuffer.load(std::memory_order_relaxed)};
 458     ALbufferlistitem *BufferLoopItem{mLoopBuffer.load(std::memory_order_relaxed)};
 459     const ALuint SampleSize{mSampleSize};
 460     const ALuint increment{mStep};
 461     if UNLIKELY(increment < 1)
 462     {
 463         /* If the voice is supposed to be stopping but can't be mixed, just
 464          * stop it before bailing.
 465          */
 466         if(vstate == Stopping)
 467             mPlayState.store(Stopped, std::memory_order_release);
 468         return;
 469     }
 470
 471     ASSUME(SampleSize > 0);
 472
 473     const size_t FrameSize{mChans.size() * SampleSize};
 474     ASSUME(FrameSize > 0);
 475
 476     ALCdevice *Device{Context->mDevice.get()};
 477     const ALuint NumSends{Device->NumAuxSends};
 478     const ALuint IrSize{Device->mHrtf ? Device->mHrtf->irSize : 0};
 479
 480     ResamplerFunc Resample{(increment == FRACTIONONE && DataPosFrac == 0) ?
 481                            Resample_<CopyTag,CTag> : mResampler};
 482
 483     ALuint Counter{(mFlags&VOICE_IS_FADING) ? SamplesToDo : 0};
 484     if(!Counter)
 485     {
 486         /* No fading, just overwrite the old/current params. */
 487         for(auto &chandata : mChans)
 488         {
 489             {
 490                 DirectParams &parms = chandata.mDryParams;
 491                 if(!(mFlags&VOICE_HAS_HRTF))
 492                     parms.Gains.Current = parms.Gains.Target;
 493                 else
 494                     parms.Hrtf.Old = parms.Hrtf.Target;
 495             }
 496             for(ALuint send{0};send < NumSends;++send)
 497             {
 498                 if(mSend[send].Buffer.empty())
 499                     continue;
 500
 501                 SendParams &parms = chandata.mWetParams[send];
 502                 parms.Gains.Current = parms.Gains.Target;
 503             }
 504         }
 505     }
 506
 507     ALuint buffers_done{0u};
 508     ALuint OutPos{0u};
 509     do {
 510         /* Figure out how many buffer samples will be needed */
 511         ALuint DstBufferSize{SamplesToDo - OutPos};
 512         ALuint SrcBufferSize;
 513
 514         if(increment <= FRACTIONONE)
 515         {
 516             /* Calculate the last written dst sample pos. */
 517             uint64_t DataSize64{DstBufferSize - 1};
 518             /* Calculate the last read src sample pos. */
 519             DataSize64 = (DataSize64*increment + DataPosFrac) >> FRACTIONBITS;
 520             /* +1 to get the src sample count, include padding. */
 521             DataSize64 += 1 + MAX_RESAMPLER_PADDING;
 522
 523             /* Result is guaranteed to be <= BUFFERSIZE+MAX_RESAMPLER_PADDING
 524              * since we won't use more src samples than dst samples+padding.
 525              */
 526             SrcBufferSize = static_cast<ALuint>(DataSize64);
 527         }
 528         else
 529         {
 530             uint64_t DataSize64{DstBufferSize};
 531             /* Calculate the end src sample pos, include padding. */
 532             DataSize64 = (DataSize64*increment + DataPosFrac) >> FRACTIONBITS;
 533             DataSize64 += MAX_RESAMPLER_PADDING;
 534
 535             if(DataSize64 <= BUFFERSIZE + MAX_RESAMPLER_PADDING)
 536                 SrcBufferSize = static_cast<ALuint>(DataSize64);
 537             else
 538             {
 539                 /* If the source size got saturated, we can't fill the desired
 540                  * dst size. Figure out how many samples we can actually mix.
 541                  */
 542                 SrcBufferSize = BUFFERSIZE + MAX_RESAMPLER_PADDING;
 543
 544                 DataSize64 = SrcBufferSize - MAX_RESAMPLER_PADDING;
 545                 DataSize64 = ((DataSize64<<FRACTIONBITS) - DataPosFrac) / increment;
 546                 if(DataSize64 < DstBufferSize)
 547                 {
 548                     /* Some mixers require being 16-byte aligned, so also limit
 549                      * to a multiple of 4 samples to maintain alignment.
 550                      */
 551                     DstBufferSize = static_cast<ALuint>(DataSize64) & ~3u;
 552                 }
 553             }
 554         }
 555
 556         if((mFlags&(VOICE_IS_CALLBACK|VOICE_CALLBACK_STOPPED)) == VOICE_IS_CALLBACK
 557             && BufferListItem)
 558         {
 559             ALbuffer *buffer{BufferListItem->mBuffer};
 560
 561             /* Exclude resampler pre-padding from the needed size. */
 562             const ALuint toLoad{SrcBufferSize - (MAX_RESAMPLER_PADDING>>1)};
 563             if(toLoad > mNumCallbackSamples)
 564             {
 565                 const size_t byteOffset{mNumCallbackSamples*FrameSize};
 566                 const size_t needBytes{toLoad*FrameSize - byteOffset};
 567
 568                 const ALsizei gotBytes{buffer->mBuffer.mCallback(buffer->mBuffer.mUserData,
 569                     &buffer->mBuffer.mData[byteOffset], static_cast<ALsizei>(needBytes))};
 570                 if(gotBytes < 1)
 571                     mFlags |= VOICE_CALLBACK_STOPPED;
 572                 else if(static_cast<ALuint>(gotBytes) < needBytes)
 573                 {
 574                     mFlags |= VOICE_CALLBACK_STOPPED;
 575                     mNumCallbackSamples += static_cast<ALuint>(static_cast<ALuint>(gotBytes) /
 576                         FrameSize);
 577                 }
 578                 else
 579                     mNumCallbackSamples = toLoad;
 580             }
 581         }
 582
 583         ASSUME(DstBufferSize > 0);
 584         for(auto &chandata : mChans)
 585         {
 586             const size_t num_chans{mChans.size()};
 587             const auto chan = static_cast<size_t>(std::distance(mChans.data(),
 588                 std::addressof(chandata)));
 589             const al::span<float> SrcData{Device->SourceData, SrcBufferSize};
 590
 591             /* Load the previous samples into the source data first, then load
 592              * what we can from the buffer queue.
 593              */
 594             auto srciter = std::copy_n(chandata.mPrevSamples.begin(), MAX_RESAMPLER_PADDING>>1,
 595                 SrcData.begin());
 596
 597             if UNLIKELY(!BufferListItem)
 598                 srciter = std::copy(chandata.mPrevSamples.begin()+(MAX_RESAMPLER_PADDING>>1),
 599                     chandata.mPrevSamples.end(), srciter);
 600             else if((mFlags&VOICE_IS_STATIC))
 601                 srciter = LoadBufferStatic(BufferListItem, BufferLoopItem, num_chans,
 602                     SampleSize, chan, DataPosInt, {srciter, SrcData.end()});
 603             else if((mFlags&VOICE_IS_CALLBACK))
 604                 srciter = LoadBufferCallback(BufferListItem, num_chans, SampleSize, chan,
 605                     mNumCallbackSamples, {srciter, SrcData.end()});
 606             else
 607                 srciter = LoadBufferQueue(BufferListItem, BufferLoopItem, num_chans,
 608                     SampleSize, chan, DataPosInt, {srciter, SrcData.end()});
 609
 610             if UNLIKELY(srciter != SrcData.end())
 611             {
 612                 /* If the source buffer wasn't filled, copy the last sample for
 613                  * the remaining buffer. Ideally it should have ended with
 614                  * silence, but if not the gain fading should help avoid clicks
 615                  * from sudden amplitude changes.
 616                  */
 617                 const float sample{*(srciter-1)};
 618                 std::fill(srciter, SrcData.end(), sample);
 619             }
 620
 621             /* Store the last source samples used for next time. */
 622             std::copy_n(&SrcData[(increment*DstBufferSize + DataPosFrac)>>FRACTIONBITS],
 623                 chandata.mPrevSamples.size(), chandata.mPrevSamples.begin());
 624
 625             /* Resample, then apply ambisonic upsampling as needed. */
 626             const float *ResampledData{Resample(&mResampleState,
 627                 &SrcData[MAX_RESAMPLER_PADDING>>1], DataPosFrac, increment,
 628                 {Device->ResampledData, DstBufferSize})};
 629             if((mFlags&VOICE_IS_AMBISONIC))
 630             {
 631                 const float hfscale{chandata.mAmbiScale};
 632                 /* Beware the evil const_cast. It's safe since it's pointing to
 633                  * either SourceData or ResampledData (both non-const), but the
 634                  * resample method takes the source as const float* and may
 635                  * return it without copying to output, making it currently
 636                  * unavoidable.
 637                  */
 638                 const al::span<float> samples{const_cast<float*>(ResampledData), DstBufferSize};
 639                 chandata.mAmbiSplitter.processHfScale(samples, hfscale);
 640             }
 641
 642             /* Now filter and mix to the appropriate outputs. */
 643             float (&FilterBuf)[BUFFERSIZE] = Device->FilteredData;
 644             {
 645                 DirectParams &parms = chandata.mDryParams;
 646                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf,
 647                     {ResampledData, DstBufferSize}, mDirect.FilterType)};
 648
 649                 if((mFlags&VOICE_HAS_HRTF))
 650                 {
 651                     const float TargetGain{UNLIKELY(vstate == Stopping) ? 0.0f :
 652                         parms.Hrtf.Target.Gain};
 653                     DoHrtfMix(samples, DstBufferSize, parms, TargetGain, Counter, OutPos, IrSize,
 654                         Device);
 655                 }
 656                 else if((mFlags&VOICE_HAS_NFC))
 657                 {
 658                     const float *TargetGains{UNLIKELY(vstate == Stopping) ? SilentTarget.data()
 659                         : parms.Gains.Target.data()};
 660                     DoNfcMix({samples, DstBufferSize}, mDirect.Buffer.data(), parms, TargetGains,
 661                         Counter, OutPos, Device);
 662                 }
 663                 else
 664                 {
 665                     const float *TargetGains{UNLIKELY(vstate == Stopping) ? SilentTarget.data()
 666                         : parms.Gains.Target.data()};
 667                     MixSamples({samples, DstBufferSize}, mDirect.Buffer,
 668                         parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 669                 }
 670             }
 671
 672             for(ALuint send{0};send < NumSends;++send)
 673             {
 674                 if(mSend[send].Buffer.empty())
 675                     continue;
 676
 677                 SendParams &parms = chandata.mWetParams[send];
 678                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf,
 679                     {ResampledData, DstBufferSize}, mSend[send].FilterType)};
 680
 681                 const float *TargetGains{UNLIKELY(vstate == Stopping) ? SilentTarget.data()
 682                     : parms.Gains.Target.data()};
 683                 MixSamples({samples, DstBufferSize}, mSend[send].Buffer,
 684                     parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 685             }
 686         }
 687         /* Update positions */
 688         DataPosFrac += increment*DstBufferSize;
 689         const ALuint SrcSamplesDone{DataPosFrac>>FRACTIONBITS};
 690         DataPosInt  += SrcSamplesDone;
 691         DataPosFrac &= FRACTIONMASK;
 692
 693         OutPos += DstBufferSize;
 694         Counter = maxu(DstBufferSize, Counter) - DstBufferSize;
 695
 696         if UNLIKELY(!BufferListItem)
 697         {
 698             /* Do nothing extra when there's no buffers. */
 699         }
 700         else if((mFlags&VOICE_IS_STATIC))
 701         {
 702             if(BufferLoopItem)
 703             {
 704                 /* Handle looping static source */
 705                 const ALbuffer *Buffer{BufferListItem->mBuffer};
 706                 const ALuint LoopStart{Buffer->LoopStart};
 707                 const ALuint LoopEnd{Buffer->LoopEnd};
 708                 if(DataPosInt >= LoopEnd)
 709                 {
 710                     assert(LoopEnd > LoopStart);
 711                     DataPosInt = ((DataPosInt-LoopStart)%(LoopEnd-LoopStart)) + LoopStart;
 712                 }
 713             }
 714             else
 715             {
 716                 /* Handle non-looping static source */
 717                 if(DataPosInt >= BufferListItem->mSampleLen)
 718                 {
 719                     BufferListItem = nullptr;
 720                     break;
 721                 }
 722             }
 723         }
 724         else if((mFlags&VOICE_IS_CALLBACK))
 725         {
 726             ALbuffer *buffer{BufferListItem->mBuffer};
 727             if(SrcSamplesDone < mNumCallbackSamples)
 728             {
 729                 const size_t byteOffset{SrcSamplesDone*FrameSize};
 730                 const size_t byteEnd{mNumCallbackSamples*FrameSize};
 731                 al::byte *data{buffer->mBuffer.mData.data()};
 732                 std::copy(data+byteOffset, data+byteEnd, data);
 733                 mNumCallbackSamples -= SrcSamplesDone;
 734             }
 735             else
 736             {
 737                 BufferListItem = nullptr;
 738                 mNumCallbackSamples = 0;
 739             }
 740         }
 741         else
 742         {
 743             /* Handle streaming source */
 744             do {
 745                 if(BufferListItem->mSampleLen > DataPosInt)
 746                     break;
 747
 748                 DataPosInt -= BufferListItem->mSampleLen;
 749
 750                 ++buffers_done;
 751                 BufferListItem = BufferListItem->mNext.load(std::memory_order_relaxed);
 752                 if(!BufferListItem) BufferListItem = BufferLoopItem;
 753             } while(BufferListItem);
 754         }
 755     } while(OutPos < SamplesToDo);
 756
 757     mFlags |= VOICE_IS_FADING;
 758
 759     /* Don't update positions and buffers if we were stopping. */
 760     if UNLIKELY(vstate == Stopping)
 761     {
 762         mPlayState.store(Stopped, std::memory_order_release);
 763         return;
 764     }
 765
 766     /* Capture the source ID in case it's reset for stopping. */
 767     const ALuint SourceID{mSourceID.load(std::memory_order_relaxed)};
 768
 769     /* Update voice info */
 770     mPosition.store(DataPosInt, std::memory_order_relaxed);
 771     mPositionFrac.store(DataPosFrac, std::memory_order_relaxed);
 772     mCurrentBuffer.store(BufferListItem, std::memory_order_relaxed);
 773     if(!BufferListItem)
 774     {
 775         mLoopBuffer.store(nullptr, std::memory_order_relaxed);
 776         mSourceID.store(0u, std::memory_order_relaxed);
 777     }
 778     std::atomic_thread_fence(std::memory_order_release);
 779
 780     /* Send any events now, after the position/buffer info was updated. */
 781     const ALbitfieldSOFT enabledevt{Context->mEnabledEvts.load(std::memory_order_acquire)};
 782     if(buffers_done > 0 && (enabledevt&EventType_BufferCompleted))
 783     {
 784         RingBuffer *ring{Context->mAsyncEvents.get()};
 785         auto evt_vec = ring->getWriteVector();
 786         if(evt_vec.first.len > 0)
 787         {
 788             AsyncEvent *evt{::new(evt_vec.first.buf) AsyncEvent{EventType_BufferCompleted}};
 789             evt->u.bufcomp.id = SourceID;
 790             evt->u.bufcomp.count = buffers_done;
 791             ring->writeAdvance(1);
 792         }
 793     }
 794
 795     if(!BufferListItem)
 796     {
 797         /* If the voice just ended, set it to Stopping so the next render
 798          * ensures any residual noise fades to 0 amplitude.
 799          */
 800         mPlayState.store(Stopping, std::memory_order_release);
 801         if((enabledevt&EventType_SourceStateChange))
 802             SendSourceStoppedEvent(Context, SourceID);
 803     }
 804 }