alc/voice.cpp

   1 /**
   2  * OpenAL cross platform audio library
   3  * Copyright (C) 1999-2007 by authors.
   4  * This library is free software; you can redistribute it and/or
   5  *  modify it under the terms of the GNU Library General Public
   6  *  License as published by the Free Software Foundation; either
   7  *  version 2 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  *  Library General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Library General Public
  15  *  License along with this library; if not, write to the
  16  *  Free Software Foundation, Inc.,
  17  *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  18  * Or go to http://www.gnu.org/copyleft/lgpl.html
  19  */
  20
  21 #include "config.h"
  22
  23 #include "voice.h"
  24
  25 #include <algorithm>
  26 #include <array>
  27 #include <atomic>
  28 #include <cassert>
  29 #include <climits>
  30 #include <cstddef>
  31 #include <cstdint>
  32 #include <iterator>
  33 #include <memory>
  34 #include <new>
  35 #include <utility>
  36
  37 #include "AL/al.h"
  38 #include "AL/alc.h"
  39
  40 #include "al/buffer.h"
  41 #include "al/event.h"
  42 #include "al/source.h"
  43 #include "alcmain.h"
  44 #include "albyte.h"
  45 #include "alconfig.h"
  46 #include "alcontext.h"
  47 #include "alnumeric.h"
  48 #include "aloptional.h"
  49 #include "alspan.h"
  50 #include "alstring.h"
  51 #include "alu.h"
  52 #include "cpu_caps.h"
  53 #include "devformat.h"
  54 #include "filters/biquad.h"
  55 #include "filters/nfc.h"
  56 #include "filters/splitter.h"
  57 #include "fmt_traits.h"
  58 #include "hrtf.h"
  59 #include "inprogext.h"
  60 #include "logging.h"
  61 #include "mixer/defs.h"
  62 #include "opthelpers.h"
  63 #include "ringbuffer.h"
  64 #include "threads.h"
  65 #include "vector.h"
  66
  67 struct CTag;
  68 #ifdef HAVE_SSE
  69 struct SSETag;
  70 #endif
  71 #ifdef HAVE_NEON
  72 struct NEONTag;
  73 #endif
  74 struct CopyTag;
  75
  76
  77 static_assert((BUFFERSIZE-1)/MAX_PITCH > 0, "MAX_PITCH is too large for BUFFERSIZE!");
  78 static_assert((INT_MAX>>FRACTIONBITS)/MAX_PITCH > BUFFERSIZE,
  79     "MAX_PITCH and/or BUFFERSIZE are too large for FRACTIONBITS!");
  80
  81
  82 Resampler ResamplerDefault{Resampler::Linear};
  83
  84 MixerFunc MixSamples{Mix_<CTag>};
  85
  86 namespace {
  87
  88 using HrtfMixerFunc = void(*)(const float *InSamples, float2 *AccumSamples, const ALuint IrSize,
  89     const MixHrtfFilter *hrtfparams, const size_t BufferSize);
  90 using HrtfMixerBlendFunc = void(*)(const float *InSamples, float2 *AccumSamples,
  91     const ALuint IrSize, const HrtfFilter *oldparams, const MixHrtfFilter *newparams,
  92     const size_t BufferSize);
  93
  94 HrtfMixerFunc MixHrtfSamples{MixHrtf_<CTag>};
  95 HrtfMixerBlendFunc MixHrtfBlendSamples{MixHrtfBlend_<CTag>};
  96
  97 inline MixerFunc SelectMixer()
  98 {
  99 #ifdef HAVE_NEON
 100     if((CPUCapFlags&CPU_CAP_NEON))
 101         return Mix_<NEONTag>;
 102 #endif
 103 #ifdef HAVE_SSE
 104     if((CPUCapFlags&CPU_CAP_SSE))
 105         return Mix_<SSETag>;
 106 #endif
 107     return Mix_<CTag>;
 108 }
 109
 110 inline HrtfMixerFunc SelectHrtfMixer()
 111 {
 112 #ifdef HAVE_NEON
 113     if((CPUCapFlags&CPU_CAP_NEON))
 114         return MixHrtf_<NEONTag>;
 115 #endif
 116 #ifdef HAVE_SSE
 117     if((CPUCapFlags&CPU_CAP_SSE))
 118         return MixHrtf_<SSETag>;
 119 #endif
 120     return MixHrtf_<CTag>;
 121 }
 122
 123 inline HrtfMixerBlendFunc SelectHrtfBlendMixer()
 124 {
 125 #ifdef HAVE_NEON
 126     if((CPUCapFlags&CPU_CAP_NEON))
 127         return MixHrtfBlend_<NEONTag>;
 128 #endif
 129 #ifdef HAVE_SSE
 130     if((CPUCapFlags&CPU_CAP_SSE))
 131         return MixHrtfBlend_<SSETag>;
 132 #endif
 133     return MixHrtfBlend_<CTag>;
 134 }
 135
 136 } // namespace
 137
 138
 139 void aluInitMixer()
 140 {
 141     if(auto resopt = ConfigValueStr(nullptr, nullptr, "resampler"))
 142     {
 143         struct ResamplerEntry {
 144             const char name[16];
 145             const Resampler resampler;
 146         };
 147         constexpr ResamplerEntry ResamplerList[]{
 148             { "none", Resampler::Point },
 149             { "point", Resampler::Point },
 150             { "cubic", Resampler::Cubic },
 151             { "bsinc12", Resampler::BSinc12 },
 152             { "fast_bsinc12", Resampler::FastBSinc12 },
 153             { "bsinc24", Resampler::BSinc24 },
 154             { "fast_bsinc24", Resampler::FastBSinc24 },
 155         };
 156
 157         const char *str{resopt->c_str()};
 158         if(al::strcasecmp(str, "bsinc") == 0)
 159         {
 160             WARN("Resampler option \"%s\" is deprecated, using bsinc12\n", str);
 161             str = "bsinc12";
 162         }
 163         else if(al::strcasecmp(str, "sinc4") == 0 || al::strcasecmp(str, "sinc8") == 0)
 164         {
 165             WARN("Resampler option \"%s\" is deprecated, using cubic\n", str);
 166             str = "cubic";
 167         }
 168
 169         auto iter = std::find_if(std::begin(ResamplerList), std::end(ResamplerList),
 170             [str](const ResamplerEntry &entry) -> bool
 171             { return al::strcasecmp(str, entry.name) == 0; });
 172         if(iter == std::end(ResamplerList))
 173             ERR("Invalid resampler: %s\n", str);
 174         else
 175             ResamplerDefault = iter->resampler;
 176     }
 177
 178     MixSamples = SelectMixer();
 179     MixHrtfBlendSamples = SelectHrtfBlendMixer();
 180     MixHrtfSamples = SelectHrtfMixer();
 181 }
 182
 183
 184 namespace {
 185
 186 void SendSourceStoppedEvent(ALCcontext *context, ALuint id)
 187 {
 188     RingBuffer *ring{context->mAsyncEvents.get()};
 189     auto evt_vec = ring->getWriteVector();
 190     if(evt_vec.first.len < 1) return;
 191
 192     AsyncEvent *evt{::new(evt_vec.first.buf) AsyncEvent{EventType_SourceStateChange}};
 193     evt->u.srcstate.id = id;
 194     evt->u.srcstate.state = AL_STOPPED;
 195
 196     ring->writeAdvance(1);
 197 }
 198
 199
 200 const float *DoFilters(BiquadFilter &lpfilter, BiquadFilter &hpfilter, float *dst,
 201     const al::span<const float> src, int type)
 202 {
 203     switch(type)
 204     {
 205     case AF_None:
 206         lpfilter.clear();
 207         hpfilter.clear();
 208         break;
 209
 210     case AF_LowPass:
 211         lpfilter.process(src, dst);
 212         hpfilter.clear();
 213         return dst;
 214     case AF_HighPass:
 215         lpfilter.clear();
 216         hpfilter.process(src, dst);
 217         return dst;
 218
 219     case AF_BandPass:
 220         DualBiquad{lpfilter, hpfilter}.process(src, dst);
 221         return dst;
 222     }
 223     return src.data();
 224 }
 225
 226
 227 void LoadSamples(float *RESTRICT dst, const al::byte *src, const size_t srcstep, FmtType srctype,
 228     const size_t samples) noexcept
 229 {
 230 #define HANDLE_FMT(T)  case T: al::LoadSampleArray<T>(dst, src, srcstep, samples); break
 231     switch(srctype)
 232     {
 233     HANDLE_FMT(FmtUByte);
 234     HANDLE_FMT(FmtShort);
 235     HANDLE_FMT(FmtFloat);
 236     HANDLE_FMT(FmtDouble);
 237     HANDLE_FMT(FmtMulaw);
 238     HANDLE_FMT(FmtAlaw);
 239     }
 240 #undef HANDLE_FMT
 241 }
 242
 243 float *LoadBufferStatic(ALbufferlistitem *BufferListItem, ALbufferlistitem *&BufferLoopItem,
 244     const size_t NumChannels, const size_t SampleSize, const size_t chan, size_t DataPosInt,
 245     al::span<float> SrcBuffer)
 246 {
 247     const ALbuffer *Buffer{BufferListItem->mBuffer};
 248     const ALuint LoopStart{Buffer->LoopStart};
 249     const ALuint LoopEnd{Buffer->LoopEnd};
 250     ASSUME(LoopEnd > LoopStart);
 251
 252     /* If current pos is beyond the loop range, do not loop */
 253     if(!BufferLoopItem || DataPosInt >= LoopEnd)
 254     {
 255         BufferLoopItem = nullptr;
 256
 257         /* Load what's left to play from the buffer */
 258         const size_t DataRem{minz(SrcBuffer.size(), Buffer->mBuffer.mSampleLen-DataPosInt)};
 259
 260         const al::byte *Data{Buffer->mBuffer.mData.data()};
 261         Data += (DataPosInt*NumChannels + chan)*SampleSize;
 262
 263         LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataRem);
 264         SrcBuffer = SrcBuffer.subspan(DataRem);
 265     }
 266     else
 267     {
 268         /* Load what's left of this loop iteration */
 269         const size_t DataRem{minz(SrcBuffer.size(), LoopEnd-DataPosInt)};
 270
 271         const al::byte *Data{Buffer->mBuffer.mData.data()};
 272         Data += (DataPosInt*NumChannels + chan)*SampleSize;
 273
 274         LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataRem);
 275         SrcBuffer = SrcBuffer.subspan(DataRem);
 276
 277         /* Load any repeats of the loop we can to fill the buffer. */
 278         const auto LoopSize = static_cast<size_t>(LoopEnd - LoopStart);
 279         while(!SrcBuffer.empty())
 280         {
 281             const size_t DataSize{minz(SrcBuffer.size(), LoopSize)};
 282
 283             Data = Buffer->mBuffer.mData.data() + (LoopStart*NumChannels + chan)*SampleSize;
 284
 285             LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataSize);
 286             SrcBuffer = SrcBuffer.subspan(DataSize);
 287         }
 288     }
 289     return SrcBuffer.begin();
 290 }
 291
 292 float *LoadBufferCallback(ALbufferlistitem *BufferListItem, const size_t NumChannels,
 293     const size_t SampleSize, const size_t chan, size_t NumCallbackSamples,
 294     al::span<float> SrcBuffer)
 295 {
 296     const ALbuffer *Buffer{BufferListItem->mBuffer};
 297
 298     /* Load what's left to play from the buffer */
 299     const size_t DataRem{minz(SrcBuffer.size(), NumCallbackSamples)};
 300
 301     const al::byte *Data{Buffer->mBuffer.mData.data() + chan*SampleSize};
 302
 303     LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataRem);
 304     SrcBuffer = SrcBuffer.subspan(DataRem);
 305
 306     return SrcBuffer.begin();
 307 }
 308
 309 float *LoadBufferQueue(ALbufferlistitem *BufferListItem, ALbufferlistitem *BufferLoopItem,
 310     const size_t NumChannels, const size_t SampleSize, const size_t chan, size_t DataPosInt,
 311     al::span<float> SrcBuffer)
 312 {
 313     /* Crawl the buffer queue to fill in the temp buffer */
 314     while(BufferListItem && !SrcBuffer.empty())
 315     {
 316         ALbuffer *Buffer{BufferListItem->mBuffer};
 317         if(!(Buffer && DataPosInt < Buffer->mBuffer.mSampleLen))
 318         {
 319             if(Buffer) DataPosInt -= Buffer->mBuffer.mSampleLen;
 320             BufferListItem = BufferListItem->mNext.load(std::memory_order_acquire);
 321             if(!BufferListItem) BufferListItem = BufferLoopItem;
 322             continue;
 323         }
 324
 325         const size_t DataSize{minz(SrcBuffer.size(), Buffer->mBuffer.mSampleLen-DataPosInt)};
 326
 327         const al::byte *Data{Buffer->mBuffer.mData.data()};
 328         Data += (DataPosInt*NumChannels + chan)*SampleSize;
 329
 330         LoadSamples(SrcBuffer.data(), Data, NumChannels, Buffer->mBuffer.mType, DataSize);
 331         SrcBuffer = SrcBuffer.subspan(DataSize);
 332         if(SrcBuffer.empty()) break;
 333
 334         DataPosInt = 0;
 335         BufferListItem = BufferListItem->mNext.load(std::memory_order_acquire);
 336         if(!BufferListItem) BufferListItem = BufferLoopItem;
 337     }
 338
 339     return SrcBuffer.begin();
 340 }
 341
 342
 343 void DoHrtfMix(const float *samples, const ALuint DstBufferSize, DirectParams &parms,
 344     const float TargetGain, const ALuint Counter, ALuint OutPos, const ALuint IrSize,
 345     ALCdevice *Device)
 346 {
 347     auto &HrtfSamples = Device->HrtfSourceData;
 348     /* Source HRTF mixing needs to include the direct delay so it remains
 349      * aligned with the direct mix's HRTF filtering.
 350      */
 351     float2 *AccumSamples{Device->HrtfAccumData + HRTF_DIRECT_DELAY};
 352
 353     /* Copy the HRTF history and new input samples into a temp buffer. */
 354     auto src_iter = std::copy(parms.Hrtf.History.begin(), parms.Hrtf.History.end(),
 355         std::begin(HrtfSamples));
 356     std::copy_n(samples, DstBufferSize, src_iter);
 357     /* Copy the last used samples back into the history buffer for later. */
 358     std::copy_n(std::begin(HrtfSamples) + DstBufferSize, parms.Hrtf.History.size(),
 359         parms.Hrtf.History.begin());
 360
 361     /* If fading and this is the first mixing pass, fade between the IRs. */
 362     ALuint fademix{0u};
 363     if(Counter && OutPos == 0)
 364     {
 365         fademix = minu(DstBufferSize, Counter);
 366
 367         float gain{TargetGain};
 368
 369         /* The new coefficients need to fade in completely since they're
 370          * replacing the old ones. To keep the gain fading consistent,
 371          * interpolate between the old and new target gains given how much of
 372          * the fade time this mix handles.
 373          */
 374         if(Counter > fademix)
 375         {
 376             const float a{static_cast<float>(fademix) / static_cast<float>(Counter)};
 377             gain = lerp(parms.Hrtf.Old.Gain, TargetGain, a);
 378         }
 379         MixHrtfFilter hrtfparams;
 380         hrtfparams.Coeffs = &parms.Hrtf.Target.Coeffs;
 381         hrtfparams.Delay = parms.Hrtf.Target.Delay;
 382         hrtfparams.Gain = 0.0f;
 383         hrtfparams.GainStep = gain / static_cast<float>(fademix);
 384
 385         MixHrtfBlendSamples(HrtfSamples, AccumSamples+OutPos, IrSize, &parms.Hrtf.Old, &hrtfparams,
 386             fademix);
 387         /* Update the old parameters with the result. */
 388         parms.Hrtf.Old = parms.Hrtf.Target;
 389         parms.Hrtf.Old.Gain = gain;
 390         OutPos += fademix;
 391     }
 392
 393     if(fademix < DstBufferSize)
 394     {
 395         const ALuint todo{DstBufferSize - fademix};
 396         float gain{TargetGain};
 397
 398         /* Interpolate the target gain if the gain fading lasts longer than
 399          * this mix.
 400          */
 401         if(Counter > DstBufferSize)
 402         {
 403             const float a{static_cast<float>(todo) / static_cast<float>(Counter-fademix)};
 404             gain = lerp(parms.Hrtf.Old.Gain, TargetGain, a);
 405         }
 406
 407         MixHrtfFilter hrtfparams;
 408         hrtfparams.Coeffs = &parms.Hrtf.Target.Coeffs;
 409         hrtfparams.Delay = parms.Hrtf.Target.Delay;
 410         hrtfparams.Gain = parms.Hrtf.Old.Gain;
 411         hrtfparams.GainStep = (gain - parms.Hrtf.Old.Gain) / static_cast<float>(todo);
 412         MixHrtfSamples(HrtfSamples+fademix, AccumSamples+OutPos, IrSize, &hrtfparams, todo);
 413         /* Store the now-current gain for next time. */
 414         parms.Hrtf.Old.Gain = gain;
 415     }
 416 }
 417
 418 void DoNfcMix(const al::span<const float> samples, FloatBufferLine *OutBuffer, DirectParams &parms,
 419     const float *TargetGains, const ALuint Counter, const ALuint OutPos, ALCdevice *Device)
 420 {
 421     using FilterProc = void (NfcFilter::*)(const al::span<const float>, float*);
 422     static constexpr FilterProc NfcProcess[MAX_AMBI_ORDER+1]{
 423         nullptr, &NfcFilter::process1, &NfcFilter::process2, &NfcFilter::process3};
 424
 425     float *CurrentGains{parms.Gains.Current.data()};
 426     MixSamples(samples, {OutBuffer, 1u}, CurrentGains, TargetGains, Counter, OutPos);
 427     ++OutBuffer;
 428     ++CurrentGains;
 429     ++TargetGains;
 430
 431     const al::span<float> nfcsamples{Device->NfcSampleData, samples.size()};
 432     size_t order{1};
 433     while(const size_t chancount{Device->NumChannelsPerOrder[order]})
 434     {
 435         (parms.NFCtrlFilter.*NfcProcess[order])(samples, nfcsamples.data());
 436         MixSamples(nfcsamples, {OutBuffer, chancount}, CurrentGains, TargetGains, Counter, OutPos);
 437         OutBuffer += chancount;
 438         CurrentGains += chancount;
 439         TargetGains += chancount;
 440         if(++order == MAX_AMBI_ORDER+1)
 441             break;
 442     }
 443 }
 444
 445 } // namespace
 446
 447 void Voice::mix(const State vstate, ALCcontext *Context, const ALuint SamplesToDo)
 448 {
 449     static constexpr std::array<float,MAX_OUTPUT_CHANNELS> SilentTarget{};
 450
 451     ASSUME(SamplesToDo > 0);
 452
 453     /* Get voice info */
 454     ALuint DataPosInt{mPosition.load(std::memory_order_relaxed)};
 455     ALuint DataPosFrac{mPositionFrac.load(std::memory_order_relaxed)};
 456     ALbufferlistitem *BufferListItem{mCurrentBuffer.load(std::memory_order_relaxed)};
 457     ALbufferlistitem *BufferLoopItem{mLoopBuffer.load(std::memory_order_relaxed)};
 458     const ALuint SampleSize{mSampleSize};
 459     const ALuint increment{mStep};
 460     if UNLIKELY(increment < 1)
 461     {
 462         /* If the voice is supposed to be stopping but can't be mixed, just
 463          * stop it before bailing.
 464          */
 465         if(vstate == Stopping)
 466             mPlayState.store(Stopped, std::memory_order_release);
 467         return;
 468     }
 469
 470     ASSUME(SampleSize > 0);
 471
 472     const size_t FrameSize{mChans.size() * SampleSize};
 473     ASSUME(FrameSize > 0);
 474
 475     ALCdevice *Device{Context->mDevice.get()};
 476     const ALuint NumSends{Device->NumAuxSends};
 477     const ALuint IrSize{Device->mHrtf ? Device->mHrtf->irSize : 0};
 478
 479     ResamplerFunc Resample{(increment == FRACTIONONE && DataPosFrac == 0) ?
 480                            Resample_<CopyTag,CTag> : mResampler};
 481
 482     ALuint Counter{(mFlags&VOICE_IS_FADING) ? SamplesToDo : 0};
 483     if(!Counter)
 484     {
 485         /* No fading, just overwrite the old/current params. */
 486         for(auto &chandata : mChans)
 487         {
 488             {
 489                 DirectParams &parms = chandata.mDryParams;
 490                 if(!(mFlags&VOICE_HAS_HRTF))
 491                     parms.Gains.Current = parms.Gains.Target;
 492                 else
 493                     parms.Hrtf.Old = parms.Hrtf.Target;
 494             }
 495             for(ALuint send{0};send < NumSends;++send)
 496             {
 497                 if(mSend[send].Buffer.empty())
 498                     continue;
 499
 500                 SendParams &parms = chandata.mWetParams[send];
 501                 parms.Gains.Current = parms.Gains.Target;
 502             }
 503         }
 504     }
 505
 506     ALuint buffers_done{0u};
 507     ALuint OutPos{0u};
 508     do {
 509         /* Figure out how many buffer samples will be needed */
 510         ALuint DstBufferSize{SamplesToDo - OutPos};
 511         ALuint SrcBufferSize;
 512
 513         if(increment <= FRACTIONONE)
 514         {
 515             /* Calculate the last written dst sample pos. */
 516             uint64_t DataSize64{DstBufferSize - 1};
 517             /* Calculate the last read src sample pos. */
 518             DataSize64 = (DataSize64*increment + DataPosFrac) >> FRACTIONBITS;
 519             /* +1 to get the src sample count, include padding. */
 520             DataSize64 += 1 + MAX_RESAMPLER_PADDING;
 521
 522             /* Result is guaranteed to be <= BUFFERSIZE+MAX_RESAMPLER_PADDING
 523              * since we won't use more src samples than dst samples+padding.
 524              */
 525             SrcBufferSize = static_cast<ALuint>(DataSize64);
 526         }
 527         else
 528         {
 529             uint64_t DataSize64{DstBufferSize};
 530             /* Calculate the end src sample pos, include padding. */
 531             DataSize64 = (DataSize64*increment + DataPosFrac) >> FRACTIONBITS;
 532             DataSize64 += MAX_RESAMPLER_PADDING;
 533
 534             if(DataSize64 <= BUFFERSIZE + MAX_RESAMPLER_PADDING)
 535                 SrcBufferSize = static_cast<ALuint>(DataSize64);
 536             else
 537             {
 538                 /* If the source size got saturated, we can't fill the desired
 539                  * dst size. Figure out how many samples we can actually mix.
 540                  */
 541                 SrcBufferSize = BUFFERSIZE + MAX_RESAMPLER_PADDING;
 542
 543                 DataSize64 = SrcBufferSize - MAX_RESAMPLER_PADDING;
 544                 DataSize64 = ((DataSize64<<FRACTIONBITS) - DataPosFrac) / increment;
 545                 if(DataSize64 < DstBufferSize)
 546                 {
 547                     /* Some mixers require being 16-byte aligned, so also limit
 548                      * to a multiple of 4 samples to maintain alignment.
 549                      */
 550                     DstBufferSize = static_cast<ALuint>(DataSize64) & ~3u;
 551                 }
 552             }
 553         }
 554
 555         if((mFlags&(VOICE_IS_CALLBACK|VOICE_CALLBACK_STOPPED)) == VOICE_IS_CALLBACK
 556             && BufferListItem)
 557         {
 558             ALbuffer *buffer{BufferListItem->mBuffer};
 559
 560             /* Exclude resampler pre-padding from the needed size. */
 561             const ALuint toLoad{SrcBufferSize - (MAX_RESAMPLER_PADDING>>1)};
 562             if(toLoad > mNumCallbackSamples)
 563             {
 564                 const size_t byteOffset{mNumCallbackSamples*FrameSize};
 565                 const size_t needBytes{toLoad*FrameSize - byteOffset};
 566
 567                 const ALsizei gotBytes{buffer->mBuffer.mCallback(buffer->mBuffer.mUserData,
 568                     &buffer->mBuffer.mData[byteOffset], static_cast<ALsizei>(needBytes))};
 569                 if(gotBytes < 1)
 570                     mFlags |= VOICE_CALLBACK_STOPPED;
 571                 else if(static_cast<ALuint>(gotBytes) < needBytes)
 572                 {
 573                     mFlags |= VOICE_CALLBACK_STOPPED;
 574                     mNumCallbackSamples += static_cast<ALuint>(static_cast<ALuint>(gotBytes) /
 575                         FrameSize);
 576                 }
 577                 else
 578                     mNumCallbackSamples = toLoad;
 579             }
 580         }
 581
 582         ASSUME(DstBufferSize > 0);
 583         for(auto &chandata : mChans)
 584         {
 585             const size_t num_chans{mChans.size()};
 586             const auto chan = static_cast<size_t>(std::distance(mChans.data(),
 587                 std::addressof(chandata)));
 588             const al::span<float> SrcData{Device->SourceData, SrcBufferSize};
 589
 590             /* Load the previous samples into the source data first, then load
 591              * what we can from the buffer queue.
 592              */
 593             auto srciter = std::copy_n(chandata.mPrevSamples.begin(), MAX_RESAMPLER_PADDING>>1,
 594                 SrcData.begin());
 595
 596             if UNLIKELY(!BufferListItem)
 597                 srciter = std::copy(chandata.mPrevSamples.begin()+(MAX_RESAMPLER_PADDING>>1),
 598                     chandata.mPrevSamples.end(), srciter);
 599             else if((mFlags&VOICE_IS_STATIC))
 600                 srciter = LoadBufferStatic(BufferListItem, BufferLoopItem, num_chans,
 601                     SampleSize, chan, DataPosInt, {srciter, SrcData.end()});
 602             else if((mFlags&VOICE_IS_CALLBACK))
 603                 srciter = LoadBufferCallback(BufferListItem, num_chans, SampleSize, chan,
 604                     mNumCallbackSamples, {srciter, SrcData.end()});
 605             else
 606                 srciter = LoadBufferQueue(BufferListItem, BufferLoopItem, num_chans,
 607                     SampleSize, chan, DataPosInt, {srciter, SrcData.end()});
 608
 609             if UNLIKELY(srciter != SrcData.end())
 610             {
 611                 /* If the source buffer wasn't filled, copy the last sample for
 612                  * the remaining buffer. Ideally it should have ended with
 613                  * silence, but if not the gain fading should help avoid clicks
 614                  * from sudden amplitude changes.
 615                  */
 616                 const float sample{*(srciter-1)};
 617                 std::fill(srciter, SrcData.end(), sample);
 618             }
 619
 620             /* Store the last source samples used for next time. */
 621             std::copy_n(&SrcData[(increment*DstBufferSize + DataPosFrac)>>FRACTIONBITS],
 622                 chandata.mPrevSamples.size(), chandata.mPrevSamples.begin());
 623
 624             /* Resample, then apply ambisonic upsampling as needed. */
 625             const float *ResampledData{Resample(&mResampleState,
 626                 &SrcData[MAX_RESAMPLER_PADDING>>1], DataPosFrac, increment,
 627                 {Device->ResampledData, DstBufferSize})};
 628             if((mFlags&VOICE_IS_AMBISONIC))
 629             {
 630                 const float hfscale{chandata.mAmbiScale};
 631                 /* Beware the evil const_cast. It's safe since it's pointing to
 632                  * either SourceData or ResampledData (both non-const), but the
 633                  * resample method takes the source as const float* and may
 634                  * return it without copying to output, making it currently
 635                  * unavoidable.
 636                  */
 637                 const al::span<float> samples{const_cast<float*>(ResampledData), DstBufferSize};
 638                 chandata.mAmbiSplitter.processHfScale(samples, hfscale);
 639             }
 640
 641             /* Now filter and mix to the appropriate outputs. */
 642             float (&FilterBuf)[BUFFERSIZE] = Device->FilteredData;
 643             {
 644                 DirectParams &parms = chandata.mDryParams;
 645                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf,
 646                     {ResampledData, DstBufferSize}, mDirect.FilterType)};
 647
 648                 if((mFlags&VOICE_HAS_HRTF))
 649                 {
 650                     const float TargetGain{UNLIKELY(vstate == Stopping) ? 0.0f :
 651                         parms.Hrtf.Target.Gain};
 652                     DoHrtfMix(samples, DstBufferSize, parms, TargetGain, Counter, OutPos, IrSize,
 653                         Device);
 654                 }
 655                 else if((mFlags&VOICE_HAS_NFC))
 656                 {
 657                     const float *TargetGains{UNLIKELY(vstate == Stopping) ? SilentTarget.data()
 658                         : parms.Gains.Target.data()};
 659                     DoNfcMix({samples, DstBufferSize}, mDirect.Buffer.data(), parms, TargetGains,
 660                         Counter, OutPos, Device);
 661                 }
 662                 else
 663                 {
 664                     const float *TargetGains{UNLIKELY(vstate == Stopping) ? SilentTarget.data()
 665                         : parms.Gains.Target.data()};
 666                     MixSamples({samples, DstBufferSize}, mDirect.Buffer,
 667                         parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 668                 }
 669             }
 670
 671             for(ALuint send{0};send < NumSends;++send)
 672             {
 673                 if(mSend[send].Buffer.empty())
 674                     continue;
 675
 676                 SendParams &parms = chandata.mWetParams[send];
 677                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf,
 678                     {ResampledData, DstBufferSize}, mSend[send].FilterType)};
 679
 680                 const float *TargetGains{UNLIKELY(vstate == Stopping) ? SilentTarget.data()
 681                     : parms.Gains.Target.data()};
 682                 MixSamples({samples, DstBufferSize}, mSend[send].Buffer,
 683                     parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 684             }
 685         }
 686         /* Update positions */
 687         DataPosFrac += increment*DstBufferSize;
 688         const ALuint SrcSamplesDone{DataPosFrac>>FRACTIONBITS};
 689         DataPosInt  += SrcSamplesDone;
 690         DataPosFrac &= FRACTIONMASK;
 691
 692         OutPos += DstBufferSize;
 693         Counter = maxu(DstBufferSize, Counter) - DstBufferSize;
 694
 695         if UNLIKELY(!BufferListItem)
 696         {
 697             /* Do nothing extra when there's no buffers. */
 698         }
 699         else if((mFlags&VOICE_IS_STATIC))
 700         {
 701             if(BufferLoopItem)
 702             {
 703                 /* Handle looping static source */
 704                 const ALbuffer *Buffer{BufferListItem->mBuffer};
 705                 const ALuint LoopStart{Buffer->LoopStart};
 706                 const ALuint LoopEnd{Buffer->LoopEnd};
 707                 if(DataPosInt >= LoopEnd)
 708                 {
 709                     assert(LoopEnd > LoopStart);
 710                     DataPosInt = ((DataPosInt-LoopStart)%(LoopEnd-LoopStart)) + LoopStart;
 711                 }
 712             }
 713             else
 714             {
 715                 /* Handle non-looping static source */
 716                 if(DataPosInt >= BufferListItem->mSampleLen)
 717                 {
 718                     BufferListItem = nullptr;
 719                     break;
 720                 }
 721             }
 722         }
 723         else if((mFlags&VOICE_IS_CALLBACK))
 724         {
 725             ALbuffer *buffer{BufferListItem->mBuffer};
 726             if(SrcSamplesDone < mNumCallbackSamples)
 727             {
 728                 const size_t byteOffset{SrcSamplesDone*FrameSize};
 729                 const size_t byteEnd{mNumCallbackSamples*FrameSize};
 730                 al::byte *data{buffer->mBuffer.mData.data()};
 731                 std::copy(data+byteOffset, data+byteEnd, data);
 732                 mNumCallbackSamples -= SrcSamplesDone;
 733             }
 734             else
 735             {
 736                 BufferListItem = nullptr;
 737                 mNumCallbackSamples = 0;
 738             }
 739         }
 740         else
 741         {
 742             /* Handle streaming source */
 743             do {
 744                 if(BufferListItem->mSampleLen > DataPosInt)
 745                     break;
 746
 747                 DataPosInt -= BufferListItem->mSampleLen;
 748
 749                 ++buffers_done;
 750                 BufferListItem = BufferListItem->mNext.load(std::memory_order_relaxed);
 751                 if(!BufferListItem) BufferListItem = BufferLoopItem;
 752             } while(BufferListItem);
 753         }
 754     } while(OutPos < SamplesToDo);
 755
 756     mFlags |= VOICE_IS_FADING;
 757
 758     /* Don't update positions and buffers if we were stopping. */
 759     if UNLIKELY(vstate == Stopping)
 760     {
 761         mPlayState.store(Stopped, std::memory_order_release);
 762         return;
 763     }
 764
 765     /* Capture the source ID in case it's reset for stopping. */
 766     const ALuint SourceID{mSourceID.load(std::memory_order_relaxed)};
 767
 768     /* Update voice info */
 769     mPosition.store(DataPosInt, std::memory_order_relaxed);
 770     mPositionFrac.store(DataPosFrac, std::memory_order_relaxed);
 771     mCurrentBuffer.store(BufferListItem, std::memory_order_relaxed);
 772     if(!BufferListItem)
 773     {
 774         mLoopBuffer.store(nullptr, std::memory_order_relaxed);
 775         mSourceID.store(0u, std::memory_order_relaxed);
 776     }
 777     std::atomic_thread_fence(std::memory_order_release);
 778
 779     /* Send any events now, after the position/buffer info was updated. */
 780     const ALbitfieldSOFT enabledevt{Context->mEnabledEvts.load(std::memory_order_acquire)};
 781     if(buffers_done > 0 && (enabledevt&EventType_BufferCompleted))
 782     {
 783         RingBuffer *ring{Context->mAsyncEvents.get()};
 784         auto evt_vec = ring->getWriteVector();
 785         if(evt_vec.first.len > 0)
 786         {
 787             AsyncEvent *evt{::new(evt_vec.first.buf) AsyncEvent{EventType_BufferCompleted}};
 788             evt->u.bufcomp.id = SourceID;
 789             evt->u.bufcomp.count = buffers_done;
 790             ring->writeAdvance(1);
 791         }
 792     }
 793
 794     if(!BufferListItem)
 795     {
 796         /* If the voice just ended, set it to Stopping so the next render
 797          * ensures any residual noise fades to 0 amplitude.
 798          */
 799         mPlayState.store(Stopping, std::memory_order_release);
 800         if((enabledevt&EventType_SourceStateChange))
 801             SendSourceStoppedEvent(Context, SourceID);
 802     }
 803 }