core/voice.cpp

   1
   2 #include "config.h"
   3
   4 #include "voice.h"
   5
   6 #include <algorithm>
   7 #include <array>
   8 #include <atomic>
   9 #include <cassert>
  10 #include <cstdint>
  11 #include <iterator>
  12 #include <memory>
  13 #include <new>
  14 #include <stdlib.h>
  15 #include <utility>
  16 #include <vector>
  17
  18 #include "albyte.h"
  19 #include "alnumeric.h"
  20 #include "aloptional.h"
  21 #include "alspan.h"
  22 #include "alstring.h"
  23 #include "ambidefs.h"
  24 #include "async_event.h"
  25 #include "buffer_storage.h"
  26 #include "context.h"
  27 #include "cpu_caps.h"
  28 #include "devformat.h"
  29 #include "device.h"
  30 #include "filters/biquad.h"
  31 #include "filters/nfc.h"
  32 #include "filters/splitter.h"
  33 #include "fmt_traits.h"
  34 #include "logging.h"
  35 #include "mixer.h"
  36 #include "mixer/defs.h"
  37 #include "mixer/hrtfdefs.h"
  38 #include "opthelpers.h"
  39 #include "resampler_limits.h"
  40 #include "ringbuffer.h"
  41 #include "vector.h"
  42 #include "voice_change.h"
  43
  44 struct CTag;
  45 #ifdef HAVE_SSE
  46 struct SSETag;
  47 #endif
  48 #ifdef HAVE_NEON
  49 struct NEONTag;
  50 #endif
  51 struct CopyTag;
  52
  53
  54 static_assert(!(sizeof(DeviceBase::MixerBufferLine)&15),
  55     "DeviceBase::MixerBufferLine must be a multiple of 16 bytes");
  56 static_assert(!(MaxResamplerEdge&3), "MaxResamplerEdge is not a multiple of 4");
  57
  58 Resampler ResamplerDefault{Resampler::Linear};
  59
  60 namespace {
  61
  62 using uint = unsigned int;
  63
  64 using HrtfMixerFunc = void(*)(const float *InSamples, float2 *AccumSamples, const uint IrSize,
  65     const MixHrtfFilter *hrtfparams, const size_t BufferSize);
  66 using HrtfMixerBlendFunc = void(*)(const float *InSamples, float2 *AccumSamples,
  67     const uint IrSize, const HrtfFilter *oldparams, const MixHrtfFilter *newparams,
  68     const size_t BufferSize);
  69
  70 HrtfMixerFunc MixHrtfSamples{MixHrtf_<CTag>};
  71 HrtfMixerBlendFunc MixHrtfBlendSamples{MixHrtfBlend_<CTag>};
  72
  73 inline MixerFunc SelectMixer()
  74 {
  75 #ifdef HAVE_NEON
  76     if((CPUCapFlags&CPU_CAP_NEON))
  77         return Mix_<NEONTag>;
  78 #endif
  79 #ifdef HAVE_SSE
  80     if((CPUCapFlags&CPU_CAP_SSE))
  81         return Mix_<SSETag>;
  82 #endif
  83     return Mix_<CTag>;
  84 }
  85
  86 inline HrtfMixerFunc SelectHrtfMixer()
  87 {
  88 #ifdef HAVE_NEON
  89     if((CPUCapFlags&CPU_CAP_NEON))
  90         return MixHrtf_<NEONTag>;
  91 #endif
  92 #ifdef HAVE_SSE
  93     if((CPUCapFlags&CPU_CAP_SSE))
  94         return MixHrtf_<SSETag>;
  95 #endif
  96     return MixHrtf_<CTag>;
  97 }
  98
  99 inline HrtfMixerBlendFunc SelectHrtfBlendMixer()
 100 {
 101 #ifdef HAVE_NEON
 102     if((CPUCapFlags&CPU_CAP_NEON))
 103         return MixHrtfBlend_<NEONTag>;
 104 #endif
 105 #ifdef HAVE_SSE
 106     if((CPUCapFlags&CPU_CAP_SSE))
 107         return MixHrtfBlend_<SSETag>;
 108 #endif
 109     return MixHrtfBlend_<CTag>;
 110 }
 111
 112 } // namespace
 113
 114 void Voice::InitMixer(al::optional<std::string> resampler)
 115 {
 116     if(resampler)
 117     {
 118         struct ResamplerEntry {
 119             const char name[16];
 120             const Resampler resampler;
 121         };
 122         constexpr ResamplerEntry ResamplerList[]{
 123             { "none", Resampler::Point },
 124             { "point", Resampler::Point },
 125             { "linear", Resampler::Linear },
 126             { "cubic", Resampler::Cubic },
 127             { "bsinc12", Resampler::BSinc12 },
 128             { "fast_bsinc12", Resampler::FastBSinc12 },
 129             { "bsinc24", Resampler::BSinc24 },
 130             { "fast_bsinc24", Resampler::FastBSinc24 },
 131         };
 132
 133         const char *str{resampler->c_str()};
 134         if(al::strcasecmp(str, "bsinc") == 0)
 135         {
 136             WARN("Resampler option \"%s\" is deprecated, using bsinc12\n", str);
 137             str = "bsinc12";
 138         }
 139         else if(al::strcasecmp(str, "sinc4") == 0 || al::strcasecmp(str, "sinc8") == 0)
 140         {
 141             WARN("Resampler option \"%s\" is deprecated, using cubic\n", str);
 142             str = "cubic";
 143         }
 144
 145         auto iter = std::find_if(std::begin(ResamplerList), std::end(ResamplerList),
 146             [str](const ResamplerEntry &entry) -> bool
 147             { return al::strcasecmp(str, entry.name) == 0; });
 148         if(iter == std::end(ResamplerList))
 149             ERR("Invalid resampler: %s\n", str);
 150         else
 151             ResamplerDefault = iter->resampler;
 152     }
 153
 154     MixSamples = SelectMixer();
 155     MixHrtfBlendSamples = SelectHrtfBlendMixer();
 156     MixHrtfSamples = SelectHrtfMixer();
 157 }
 158
 159
 160 namespace {
 161
 162 void SendSourceStoppedEvent(ContextBase *context, uint id)
 163 {
 164     RingBuffer *ring{context->mAsyncEvents.get()};
 165     auto evt_vec = ring->getWriteVector();
 166     if(evt_vec.first.len < 1) return;
 167
 168     AsyncEvent *evt{al::construct_at(reinterpret_cast<AsyncEvent*>(evt_vec.first.buf),
 169         AsyncEvent::SourceStateChange)};
 170     evt->u.srcstate.id = id;
 171     evt->u.srcstate.state = AsyncEvent::SrcState::Stop;
 172
 173     ring->writeAdvance(1);
 174 }
 175
 176
 177 const float *DoFilters(BiquadFilter &lpfilter, BiquadFilter &hpfilter, float *dst,
 178     const al::span<const float> src, int type)
 179 {
 180     switch(type)
 181     {
 182     case AF_None:
 183         lpfilter.clear();
 184         hpfilter.clear();
 185         break;
 186
 187     case AF_LowPass:
 188         lpfilter.process(src, dst);
 189         hpfilter.clear();
 190         return dst;
 191     case AF_HighPass:
 192         lpfilter.clear();
 193         hpfilter.process(src, dst);
 194         return dst;
 195
 196     case AF_BandPass:
 197         DualBiquad{lpfilter, hpfilter}.process(src, dst);
 198         return dst;
 199     }
 200     return src.data();
 201 }
 202
 203
 204 template<FmtType Type>
 205 inline void LoadSamples(const al::span<float*> dstSamples, const size_t dstOffset,
 206     const al::byte *src, const size_t srcOffset, const FmtChannels srcChans, const size_t srcStep,
 207     const size_t samples) noexcept
 208 {
 209     constexpr size_t sampleSize{sizeof(typename al::FmtTypeTraits<Type>::Type)};
 210     auto s = src + srcOffset*srcStep*sampleSize;
 211     if(srcChans == FmtUHJ2 || srcChans == FmtSuperStereo)
 212     {
 213         al::LoadSampleArray<Type>(dstSamples[0]+dstOffset, s, srcStep, samples);
 214         al::LoadSampleArray<Type>(dstSamples[1]+dstOffset, s+sampleSize, srcStep, samples);
 215         std::fill_n(dstSamples[2]+dstOffset, samples, 0.0f);
 216     }
 217     else
 218     {
 219         for(auto *dst : dstSamples)
 220         {
 221             al::LoadSampleArray<Type>(dst+dstOffset, s, srcStep, samples);
 222             s += sampleSize;
 223         }
 224     }
 225 }
 226
 227 void LoadSamples(const al::span<float*> dstSamples, const size_t dstOffset, const al::byte *src,
 228     const size_t srcOffset, const FmtType srcType, const FmtChannels srcChans,
 229     const size_t srcStep, const size_t samples) noexcept
 230 {
 231 #define HANDLE_FMT(T) case T:                                                 \
 232     LoadSamples<T>(dstSamples, dstOffset, src, srcOffset, srcChans, srcStep,  \
 233         samples);                                                             \
 234     break
 235
 236     switch(srcType)
 237     {
 238     HANDLE_FMT(FmtUByte);
 239     HANDLE_FMT(FmtShort);
 240     HANDLE_FMT(FmtFloat);
 241     HANDLE_FMT(FmtDouble);
 242     HANDLE_FMT(FmtMulaw);
 243     HANDLE_FMT(FmtAlaw);
 244     }
 245 #undef HANDLE_FMT
 246 }
 247
 248 void LoadBufferStatic(VoiceBufferItem *buffer, VoiceBufferItem *bufferLoopItem,
 249     const size_t dataPosInt, const FmtType sampleType, const FmtChannels sampleChannels,
 250     const size_t srcStep, const size_t samplesToLoad, const al::span<float*> voiceSamples)
 251 {
 252     const uint loopStart{buffer->mLoopStart};
 253     const uint loopEnd{buffer->mLoopEnd};
 254     ASSUME(loopEnd > loopStart);
 255
 256     /* If current pos is beyond the loop range, do not loop */
 257     if(!bufferLoopItem || dataPosInt >= loopEnd)
 258     {
 259         /* Load what's left to play from the buffer */
 260         const size_t remaining{minz(samplesToLoad, buffer->mSampleLen-dataPosInt)};
 261         LoadSamples(voiceSamples, 0, buffer->mSamples, dataPosInt, sampleType, sampleChannels,
 262             srcStep, remaining);
 263
 264         if(const size_t toFill{samplesToLoad - remaining})
 265         {
 266             for(auto *chanbuffer : voiceSamples)
 267             {
 268                 auto srcsamples = chanbuffer + remaining - 1;
 269                 std::fill_n(srcsamples + 1, toFill, *srcsamples);
 270             }
 271         }
 272     }
 273     else
 274     {
 275         /* Load what's left of this loop iteration */
 276         const size_t remaining{minz(samplesToLoad, loopEnd-dataPosInt)};
 277         LoadSamples(voiceSamples, 0, buffer->mSamples, dataPosInt, sampleType, sampleChannels,
 278             srcStep, remaining);
 279
 280         /* Load repeats of the loop to fill the buffer. */
 281         const auto loopSize = static_cast<size_t>(loopEnd - loopStart);
 282         size_t samplesLoaded{remaining};
 283         while(const size_t toFill{minz(samplesToLoad - samplesLoaded, loopSize)})
 284         {
 285             LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, loopStart, sampleType,
 286                 sampleChannels, srcStep, toFill);
 287             samplesLoaded += toFill;
 288         }
 289     }
 290 }
 291
 292 void LoadBufferCallback(VoiceBufferItem *buffer, const size_t numCallbackSamples,
 293     const FmtType sampleType, const FmtChannels sampleChannels, const size_t srcStep,
 294     const size_t samplesToLoad, const al::span<float*> voiceSamples)
 295 {
 296     /* Load what's left to play from the buffer */
 297     const size_t remaining{minz(samplesToLoad, numCallbackSamples)};
 298     LoadSamples(voiceSamples, 0, buffer->mSamples, 0, sampleType, sampleChannels, srcStep,
 299         remaining);
 300
 301     if(const size_t toFill{samplesToLoad - remaining})
 302     {
 303         for(auto *chanbuffer : voiceSamples)
 304         {
 305             auto srcsamples = chanbuffer + remaining - 1;
 306             std::fill_n(srcsamples + 1, toFill, *srcsamples);
 307         }
 308     }
 309 }
 310
 311 void LoadBufferQueue(VoiceBufferItem *buffer, VoiceBufferItem *bufferLoopItem,
 312     size_t dataPosInt, const FmtType sampleType, const FmtChannels sampleChannels,
 313     const size_t srcStep, const size_t samplesToLoad, const al::span<float*> voiceSamples)
 314 {
 315     /* Crawl the buffer queue to fill in the temp buffer */
 316     size_t samplesLoaded{0};
 317     while(buffer && samplesLoaded != samplesToLoad)
 318     {
 319         if(dataPosInt >= buffer->mSampleLen)
 320         {
 321             dataPosInt -= buffer->mSampleLen;
 322             buffer = buffer->mNext.load(std::memory_order_acquire);
 323             if(!buffer) buffer = bufferLoopItem;
 324             continue;
 325         }
 326
 327         const size_t remaining{minz(samplesToLoad-samplesLoaded, buffer->mSampleLen-dataPosInt)};
 328         LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, dataPosInt, sampleType,
 329             sampleChannels, srcStep, remaining);
 330
 331         samplesLoaded += remaining;
 332         if(samplesLoaded == samplesToLoad)
 333             break;
 334
 335         dataPosInt = 0;
 336         buffer = buffer->mNext.load(std::memory_order_acquire);
 337         if(!buffer) buffer = bufferLoopItem;
 338     }
 339     if(const size_t toFill{samplesToLoad - samplesLoaded})
 340     {
 341         size_t chanidx{0};
 342         for(auto *chanbuffer : voiceSamples)
 343         {
 344             auto srcsamples = chanbuffer + samplesLoaded - 1;
 345             std::fill_n(srcsamples + 1, toFill, *srcsamples);
 346             ++chanidx;
 347         }
 348     }
 349 }
 350
 351
 352 void DoHrtfMix(const float *samples, const uint DstBufferSize, DirectParams &parms,
 353     const float TargetGain, const uint Counter, uint OutPos, const bool IsPlaying,
 354     DeviceBase *Device)
 355 {
 356     const uint IrSize{Device->mIrSize};
 357     auto &HrtfSamples = Device->HrtfSourceData;
 358     auto &AccumSamples = Device->HrtfAccumData;
 359
 360     /* Copy the HRTF history and new input samples into a temp buffer. */
 361     auto src_iter = std::copy(parms.Hrtf.History.begin(), parms.Hrtf.History.end(),
 362         std::begin(HrtfSamples));
 363     std::copy_n(samples, DstBufferSize, src_iter);
 364     /* Copy the last used samples back into the history buffer for later. */
 365     if(likely(IsPlaying))
 366         std::copy_n(std::begin(HrtfSamples) + DstBufferSize, parms.Hrtf.History.size(),
 367             parms.Hrtf.History.begin());
 368
 369     /* If fading and this is the first mixing pass, fade between the IRs. */
 370     uint fademix{0u};
 371     if(Counter && OutPos == 0)
 372     {
 373         fademix = minu(DstBufferSize, Counter);
 374
 375         float gain{TargetGain};
 376
 377         /* The new coefficients need to fade in completely since they're
 378          * replacing the old ones. To keep the gain fading consistent,
 379          * interpolate between the old and new target gains given how much of
 380          * the fade time this mix handles.
 381          */
 382         if(Counter > fademix)
 383         {
 384             const float a{static_cast<float>(fademix) / static_cast<float>(Counter)};
 385             gain = lerpf(parms.Hrtf.Old.Gain, TargetGain, a);
 386         }
 387
 388         MixHrtfFilter hrtfparams{
 389             parms.Hrtf.Target.Coeffs,
 390             parms.Hrtf.Target.Delay,
 391             0.0f, gain / static_cast<float>(fademix)};
 392         MixHrtfBlendSamples(HrtfSamples, AccumSamples+OutPos, IrSize, &parms.Hrtf.Old, &hrtfparams,
 393             fademix);
 394
 395         /* Update the old parameters with the result. */
 396         parms.Hrtf.Old = parms.Hrtf.Target;
 397         parms.Hrtf.Old.Gain = gain;
 398         OutPos += fademix;
 399     }
 400
 401     if(fademix < DstBufferSize)
 402     {
 403         const uint todo{DstBufferSize - fademix};
 404         float gain{TargetGain};
 405
 406         /* Interpolate the target gain if the gain fading lasts longer than
 407          * this mix.
 408          */
 409         if(Counter > DstBufferSize)
 410         {
 411             const float a{static_cast<float>(todo) / static_cast<float>(Counter-fademix)};
 412             gain = lerpf(parms.Hrtf.Old.Gain, TargetGain, a);
 413         }
 414
 415         MixHrtfFilter hrtfparams{
 416             parms.Hrtf.Target.Coeffs,
 417             parms.Hrtf.Target.Delay,
 418             parms.Hrtf.Old.Gain,
 419             (gain - parms.Hrtf.Old.Gain) / static_cast<float>(todo)};
 420         MixHrtfSamples(HrtfSamples+fademix, AccumSamples+OutPos, IrSize, &hrtfparams, todo);
 421
 422         /* Store the now-current gain for next time. */
 423         parms.Hrtf.Old.Gain = gain;
 424     }
 425 }
 426
 427 void DoNfcMix(const al::span<const float> samples, FloatBufferLine *OutBuffer, DirectParams &parms,
 428     const float *TargetGains, const uint Counter, const uint OutPos, DeviceBase *Device)
 429 {
 430     using FilterProc = void (NfcFilter::*)(const al::span<const float>, float*);
 431     static constexpr FilterProc NfcProcess[MaxAmbiOrder+1]{
 432         nullptr, &NfcFilter::process1, &NfcFilter::process2, &NfcFilter::process3};
 433
 434     float *CurrentGains{parms.Gains.Current.data()};
 435     MixSamples(samples, {OutBuffer, 1u}, CurrentGains, TargetGains, Counter, OutPos);
 436     ++OutBuffer;
 437     ++CurrentGains;
 438     ++TargetGains;
 439
 440     const al::span<float> nfcsamples{Device->NfcSampleData, samples.size()};
 441     size_t order{1};
 442     while(const size_t chancount{Device->NumChannelsPerOrder[order]})
 443     {
 444         (parms.NFCtrlFilter.*NfcProcess[order])(samples, nfcsamples.data());
 445         MixSamples(nfcsamples, {OutBuffer, chancount}, CurrentGains, TargetGains, Counter, OutPos);
 446         OutBuffer += chancount;
 447         CurrentGains += chancount;
 448         TargetGains += chancount;
 449         if(++order == MaxAmbiOrder+1)
 450             break;
 451     }
 452 }
 453
 454 } // namespace
 455
 456 void Voice::mix(const State vstate, ContextBase *Context, const uint SamplesToDo)
 457 {
 458     static constexpr std::array<float,MAX_OUTPUT_CHANNELS> SilentTarget{};
 459
 460     ASSUME(SamplesToDo > 0);
 461
 462     /* Get voice info */
 463     uint DataPosInt{mPosition.load(std::memory_order_relaxed)};
 464     uint DataPosFrac{mPositionFrac.load(std::memory_order_relaxed)};
 465     VoiceBufferItem *BufferListItem{mCurrentBuffer.load(std::memory_order_relaxed)};
 466     VoiceBufferItem *BufferLoopItem{mLoopBuffer.load(std::memory_order_relaxed)};
 467     const uint increment{mStep};
 468     if UNLIKELY(increment < 1)
 469     {
 470         /* If the voice is supposed to be stopping but can't be mixed, just
 471          * stop it before bailing.
 472          */
 473         if(vstate == Stopping)
 474             mPlayState.store(Stopped, std::memory_order_release);
 475         return;
 476     }
 477
 478     DeviceBase *Device{Context->mDevice};
 479     const uint NumSends{Device->NumAuxSends};
 480
 481     ResamplerFunc Resample{(increment == MixerFracOne && DataPosFrac == 0) ?
 482                            Resample_<CopyTag,CTag> : mResampler};
 483
 484     uint Counter{mFlags.test(VoiceIsFading) ? SamplesToDo : 0};
 485     if(!Counter)
 486     {
 487         /* No fading, just overwrite the old/current params. */
 488         for(auto &chandata : mChans)
 489         {
 490             {
 491                 DirectParams &parms = chandata.mDryParams;
 492                 if(!mFlags.test(VoiceHasHrtf))
 493                     parms.Gains.Current = parms.Gains.Target;
 494                 else
 495                     parms.Hrtf.Old = parms.Hrtf.Target;
 496             }
 497             for(uint send{0};send < NumSends;++send)
 498             {
 499                 if(mSend[send].Buffer.empty())
 500                     continue;
 501
 502                 SendParams &parms = chandata.mWetParams[send];
 503                 parms.Gains.Current = parms.Gains.Target;
 504             }
 505         }
 506     }
 507     else if UNLIKELY(!BufferListItem)
 508         Counter = std::min(Counter, 64u);
 509
 510     std::array<float*,DeviceBase::MixerChannelsMax> SamplePointers;
 511     const al::span<float*> MixingSamples{SamplePointers.data(), mChans.size()};
 512     auto offset_bufferline = [](DeviceBase::MixerBufferLine &bufline) noexcept -> float*
 513     { return bufline.data() + MaxResamplerEdge; };
 514     std::transform(Device->mSampleData.end() - mChans.size(), Device->mSampleData.end(),
 515         MixingSamples.begin(), offset_bufferline);
 516
 517     const uint PostPadding{MaxResamplerEdge + mDecoderPadding};
 518     uint buffers_done{0u};
 519     uint OutPos{0u};
 520     do {
 521         /* Figure out how many buffer samples will be needed */
 522         uint DstBufferSize{SamplesToDo - OutPos};
 523         uint SrcBufferSize;
 524
 525         if(increment <= MixerFracOne)
 526         {
 527             /* Calculate the last written dst sample pos. */
 528             uint64_t DataSize64{DstBufferSize - 1};
 529             /* Calculate the last read src sample pos. */
 530             DataSize64 = (DataSize64*increment + DataPosFrac) >> MixerFracBits;
 531             /* +1 to get the src sample count, include padding. */
 532             DataSize64 += 1 + PostPadding;
 533
 534             /* Result is guaranteed to be <= BufferLineSize+PostPadding since
 535              * we won't use more src samples than dst samples+padding.
 536              */
 537             SrcBufferSize = static_cast<uint>(DataSize64);
 538         }
 539         else
 540         {
 541             uint64_t DataSize64{DstBufferSize};
 542             /* Calculate the end src sample pos, include padding. */
 543             DataSize64 = (DataSize64*increment + DataPosFrac) >> MixerFracBits;
 544             DataSize64 += PostPadding;
 545
 546             if(DataSize64 <= DeviceBase::MixerLineSize - MaxResamplerEdge)
 547                 SrcBufferSize = static_cast<uint>(DataSize64);
 548             else
 549             {
 550                 /* If the source size got saturated, we can't fill the desired
 551                  * dst size. Figure out how many samples we can actually mix.
 552                  */
 553                 SrcBufferSize = DeviceBase::MixerLineSize - MaxResamplerEdge;
 554
 555                 DataSize64 = SrcBufferSize - PostPadding;
 556                 DataSize64 = ((DataSize64<<MixerFracBits) - DataPosFrac) / increment;
 557                 if(DataSize64 < DstBufferSize)
 558                 {
 559                     /* Some mixers require being 16-byte aligned, so also limit
 560                      * to a multiple of 4 samples to maintain alignment.
 561                      */
 562                     DstBufferSize = static_cast<uint>(DataSize64) & ~3u;
 563                     /* If the voice is stopping, only one mixing iteration will
 564                      * be done, so ensure it fades out completely this mix.
 565                      */
 566                     if(unlikely(vstate == Stopping))
 567                         Counter = std::min(Counter, DstBufferSize);
 568                 }
 569                 ASSUME(DstBufferSize > 0);
 570             }
 571         }
 572
 573         if(unlikely(!BufferListItem))
 574         {
 575             const size_t srcOffset{(increment*DstBufferSize + DataPosFrac)>>MixerFracBits};
 576             auto prevSamples = mPrevSamples.data();
 577             SrcBufferSize = SrcBufferSize - PostPadding + MaxResamplerEdge;
 578             for(auto *chanbuffer : MixingSamples)
 579             {
 580                 auto srcend = std::copy_n(prevSamples->data(), MaxResamplerPadding,
 581                     chanbuffer-MaxResamplerEdge);
 582
 583                 /* When loading from a voice that ended prematurely, only take
 584                  * the samples that get closest to 0 amplitude. This helps
 585                  * certain sounds fade out better.
 586                  */
 587                 auto abs_lt = [](const float lhs, const float rhs) noexcept -> bool
 588                 { return std::abs(lhs) < std::abs(rhs); };
 589                 auto srciter = std::min_element(chanbuffer, srcend, abs_lt);
 590
 591                 std::fill(srciter+1, chanbuffer + SrcBufferSize, *srciter);
 592
 593                 std::copy_n(chanbuffer-MaxResamplerEdge+srcOffset, prevSamples->size(),
 594                     prevSamples->data());
 595                 ++prevSamples;
 596             }
 597         }
 598         else
 599         {
 600             auto prevSamples = mPrevSamples.data();
 601             for(auto *chanbuffer : MixingSamples)
 602             {
 603                 std::copy_n(prevSamples->data(), MaxResamplerEdge, chanbuffer-MaxResamplerEdge);
 604                 ++prevSamples;
 605             }
 606             if(mFlags.test(VoiceIsStatic))
 607                 LoadBufferStatic(BufferListItem, BufferLoopItem, DataPosInt, mFmtType,
 608                     mFmtChannels, mFrameStep, SrcBufferSize, MixingSamples);
 609             else if(mFlags.test(VoiceIsCallback))
 610             {
 611                 if(!mFlags.test(VoiceCallbackStopped) && SrcBufferSize > mNumCallbackSamples)
 612                 {
 613                     const size_t byteOffset{mNumCallbackSamples*mFrameSize};
 614                     const size_t needBytes{SrcBufferSize*mFrameSize - byteOffset};
 615
 616                     const int gotBytes{BufferListItem->mCallback(BufferListItem->mUserData,
 617                         &BufferListItem->mSamples[byteOffset], static_cast<int>(needBytes))};
 618                     if(gotBytes < 0)
 619                         mFlags.set(VoiceCallbackStopped);
 620                     else if(static_cast<uint>(gotBytes) < needBytes)
 621                     {
 622                         mFlags.set(VoiceCallbackStopped);
 623                         mNumCallbackSamples += static_cast<uint>(gotBytes) / mFrameSize;
 624                     }
 625                     else
 626                         mNumCallbackSamples = SrcBufferSize;
 627                 }
 628                 LoadBufferCallback(BufferListItem, mNumCallbackSamples, mFmtType, mFmtChannels,
 629                     mFrameStep, SrcBufferSize, MixingSamples);
 630             }
 631             else
 632                 LoadBufferQueue(BufferListItem, BufferLoopItem, DataPosInt, mFmtType, mFmtChannels,
 633                     mFrameStep, SrcBufferSize, MixingSamples);
 634
 635             const size_t srcOffset{(increment*DstBufferSize + DataPosFrac)>>MixerFracBits};
 636             if(mDecoder)
 637             {
 638                 SrcBufferSize = SrcBufferSize - PostPadding + MaxResamplerEdge;
 639                 mDecoder->decode(MixingSamples, SrcBufferSize,
 640                     likely(vstate == Playing) ? srcOffset : 0);
 641             }
 642             /* Store the last source samples used for next time. */
 643             if(likely(vstate == Playing))
 644             {
 645                 prevSamples = mPrevSamples.data();
 646                 for(auto *chanbuffer : MixingSamples)
 647                 {
 648                     /* Store the last source samples used for next time. */
 649                     std::copy_n(chanbuffer-MaxResamplerEdge+srcOffset, prevSamples->size(),
 650                         prevSamples->data());
 651                     ++prevSamples;
 652                 }
 653             }
 654         }
 655
 656         auto voiceSamples = MixingSamples.begin();
 657         for(auto &chandata : mChans)
 658         {
 659             /* Resample, then apply ambisonic upsampling as needed. */
 660             float *ResampledData{Resample(&mResampleState, *voiceSamples, DataPosFrac, increment,
 661                 {Device->ResampledData, DstBufferSize})};
 662             ++voiceSamples;
 663
 664             if(mFlags.test(VoiceIsAmbisonic))
 665                 chandata.mAmbiSplitter.processScale({ResampledData, DstBufferSize},
 666                     chandata.mAmbiHFScale, chandata.mAmbiLFScale);
 667
 668             /* Now filter and mix to the appropriate outputs. */
 669             const al::span<float,BufferLineSize> FilterBuf{Device->FilteredData};
 670             {
 671                 DirectParams &parms = chandata.mDryParams;
 672                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf.data(),
 673                     {ResampledData, DstBufferSize}, mDirect.FilterType)};
 674
 675                 if(mFlags.test(VoiceHasHrtf))
 676                 {
 677                     const float TargetGain{parms.Hrtf.Target.Gain * likely(vstate == Playing)};
 678                     DoHrtfMix(samples, DstBufferSize, parms, TargetGain, Counter, OutPos,
 679                         (vstate == Playing), Device);
 680                 }
 681                 else
 682                 {
 683                     const float *TargetGains{likely(vstate == Playing) ? parms.Gains.Target.data()
 684                         : SilentTarget.data()};
 685                     if(mFlags.test(VoiceHasNfc))
 686                         DoNfcMix({samples, DstBufferSize}, mDirect.Buffer.data(), parms,
 687                             TargetGains, Counter, OutPos, Device);
 688                     else
 689                         MixSamples({samples, DstBufferSize}, mDirect.Buffer,
 690                             parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 691                 }
 692             }
 693
 694             for(uint send{0};send < NumSends;++send)
 695             {
 696                 if(mSend[send].Buffer.empty())
 697                     continue;
 698
 699                 SendParams &parms = chandata.mWetParams[send];
 700                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf.data(),
 701                     {ResampledData, DstBufferSize}, mSend[send].FilterType)};
 702
 703                 const float *TargetGains{likely(vstate == Playing) ? parms.Gains.Target.data()
 704                     : SilentTarget.data()};
 705                 MixSamples({samples, DstBufferSize}, mSend[send].Buffer,
 706                     parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 707             }
 708         }
 709         /* If the voice is stopping, we're now done. */
 710         if(unlikely(vstate == Stopping))
 711             break;
 712
 713         /* Update positions */
 714         DataPosFrac += increment*DstBufferSize;
 715         const uint SrcSamplesDone{DataPosFrac>>MixerFracBits};
 716         DataPosInt  += SrcSamplesDone;
 717         DataPosFrac &= MixerFracMask;
 718
 719         OutPos += DstBufferSize;
 720         Counter = maxu(DstBufferSize, Counter) - DstBufferSize;
 721
 722         if(unlikely(!BufferListItem))
 723         {
 724             /* Do nothing extra when there's no buffers. */
 725         }
 726         else if(mFlags.test(VoiceIsStatic))
 727         {
 728             if(BufferLoopItem)
 729             {
 730                 /* Handle looping static source */
 731                 const uint LoopStart{BufferListItem->mLoopStart};
 732                 const uint LoopEnd{BufferListItem->mLoopEnd};
 733                 if(DataPosInt >= LoopEnd)
 734                 {
 735                     assert(LoopEnd > LoopStart);
 736                     DataPosInt = ((DataPosInt-LoopStart)%(LoopEnd-LoopStart)) + LoopStart;
 737                 }
 738             }
 739             else
 740             {
 741                 /* Handle non-looping static source */
 742                 if(DataPosInt >= BufferListItem->mSampleLen)
 743                 {
 744                     BufferListItem = nullptr;
 745                     break;
 746                 }
 747             }
 748         }
 749         else if(mFlags.test(VoiceIsCallback))
 750         {
 751             /* Handle callback buffer source */
 752             if(SrcSamplesDone < mNumCallbackSamples)
 753             {
 754                 const size_t byteOffset{SrcSamplesDone*mFrameSize};
 755                 const size_t byteEnd{mNumCallbackSamples*mFrameSize};
 756                 al::byte *data{BufferListItem->mSamples};
 757                 std::copy(data+byteOffset, data+byteEnd, data);
 758                 mNumCallbackSamples -= SrcSamplesDone;
 759             }
 760             else
 761             {
 762                 BufferListItem = nullptr;
 763                 mNumCallbackSamples = 0;
 764             }
 765         }
 766         else
 767         {
 768             /* Handle streaming source */
 769             do {
 770                 if(BufferListItem->mSampleLen > DataPosInt)
 771                     break;
 772
 773                 DataPosInt -= BufferListItem->mSampleLen;
 774
 775                 ++buffers_done;
 776                 BufferListItem = BufferListItem->mNext.load(std::memory_order_relaxed);
 777                 if(!BufferListItem) BufferListItem = BufferLoopItem;
 778             } while(BufferListItem);
 779         }
 780     } while(OutPos < SamplesToDo);
 781
 782     mFlags.set(VoiceIsFading);
 783
 784     /* Don't update positions and buffers if we were stopping. */
 785     if(unlikely(vstate == Stopping))
 786     {
 787         mPlayState.store(Stopped, std::memory_order_release);
 788         return;
 789     }
 790
 791     /* Capture the source ID in case it's reset for stopping. */
 792     const uint SourceID{mSourceID.load(std::memory_order_relaxed)};
 793
 794     /* Update voice info */
 795     mPosition.store(DataPosInt, std::memory_order_relaxed);
 796     mPositionFrac.store(DataPosFrac, std::memory_order_relaxed);
 797     mCurrentBuffer.store(BufferListItem, std::memory_order_relaxed);
 798     if(!BufferListItem)
 799     {
 800         mLoopBuffer.store(nullptr, std::memory_order_relaxed);
 801         mSourceID.store(0u, std::memory_order_relaxed);
 802     }
 803     std::atomic_thread_fence(std::memory_order_release);
 804
 805     /* Send any events now, after the position/buffer info was updated. */
 806     const uint enabledevt{Context->mEnabledEvts.load(std::memory_order_acquire)};
 807     if(buffers_done > 0 && (enabledevt&AsyncEvent::BufferCompleted))
 808     {
 809         RingBuffer *ring{Context->mAsyncEvents.get()};
 810         auto evt_vec = ring->getWriteVector();
 811         if(evt_vec.first.len > 0)
 812         {
 813             AsyncEvent *evt{al::construct_at(reinterpret_cast<AsyncEvent*>(evt_vec.first.buf),
 814                 AsyncEvent::BufferCompleted)};
 815             evt->u.bufcomp.id = SourceID;
 816             evt->u.bufcomp.count = buffers_done;
 817             ring->writeAdvance(1);
 818         }
 819     }
 820
 821     if(!BufferListItem)
 822     {
 823         /* If the voice just ended, set it to Stopping so the next render
 824          * ensures any residual noise fades to 0 amplitude.
 825          */
 826         mPlayState.store(Stopping, std::memory_order_release);
 827         if((enabledevt&AsyncEvent::SourceStateChange))
 828             SendSourceStoppedEvent(Context, SourceID);
 829     }
 830 }
 831
 832 void Voice::prepare(DeviceBase *device)
 833 {
 834     /* Even if storing really high order ambisonics, we only mix channels for
 835      * orders up to the device order. The rest are simply dropped.
 836      */
 837     uint num_channels{(mFmtChannels == FmtUHJ2 || mFmtChannels == FmtSuperStereo) ? 3 :
 838         ChannelsFromFmt(mFmtChannels, minu(mAmbiOrder, device->mAmbiOrder))};
 839     if(unlikely(num_channels > device->mSampleData.size()))
 840     {
 841         ERR("Unexpected channel count: %u (limit: %zu, %d:%d)\n", num_channels,
 842             device->mSampleData.size(), mFmtChannels, mAmbiOrder);
 843         num_channels = static_cast<uint>(device->mSampleData.size());
 844     }
 845     if(mChans.capacity() > 2 && num_channels < mChans.capacity())
 846     {
 847         decltype(mChans){}.swap(mChans);
 848         decltype(mPrevSamples){}.swap(mPrevSamples);
 849     }
 850     mChans.reserve(maxu(2, num_channels));
 851     mChans.resize(num_channels);
 852     mPrevSamples.reserve(maxu(2, num_channels));
 853     mPrevSamples.resize(num_channels);
 854
 855     if(mFmtChannels == FmtSuperStereo)
 856     {
 857         mDecoder = std::make_unique<UhjStereoDecoder>();
 858         mDecoderPadding = UhjStereoDecoder::sFilterDelay;
 859     }
 860     else if(IsUHJ(mFmtChannels))
 861     {
 862         mDecoder = std::make_unique<UhjDecoder>();
 863         mDecoderPadding = UhjDecoder::sFilterDelay;
 864     }
 865     else
 866     {
 867         mDecoder = nullptr;
 868         mDecoderPadding = 0;
 869     }
 870
 871     /* Clear the stepping value explicitly so the mixer knows not to mix this
 872      * until the update gets applied.
 873      */
 874     mStep = 0;
 875
 876     /* Make sure the sample history is cleared. */
 877     std::fill(mPrevSamples.begin(), mPrevSamples.end(), HistoryLine{});
 878
 879     /* Don't need to set the VoiceIsAmbisonic flag if the device is not higher
 880      * order than the voice. No HF scaling is necessary to mix it.
 881      */
 882     if(mAmbiOrder && device->mAmbiOrder > mAmbiOrder)
 883     {
 884         const uint8_t *OrderFromChan{Is2DAmbisonic(mFmtChannels) ?
 885             AmbiIndex::OrderFrom2DChannel().data() : AmbiIndex::OrderFromChannel().data()};
 886         const auto scales = AmbiScale::GetHFOrderScales(mAmbiOrder, device->mAmbiOrder);
 887
 888         const BandSplitter splitter{device->mXOverFreq / static_cast<float>(device->Frequency)};
 889         for(auto &chandata : mChans)
 890         {
 891             chandata.mAmbiHFScale = scales[*(OrderFromChan++)];
 892             chandata.mAmbiLFScale = 1.0f;
 893             chandata.mAmbiSplitter = splitter;
 894             chandata.mDryParams = DirectParams{};
 895             chandata.mDryParams.NFCtrlFilter = device->mNFCtrlFilter;
 896             std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
 897         }
 898         /* 2-channel UHJ needs different shelf filters. However, we can't just
 899          * use different shelf filters after mixing it and with any old speaker
 900          * setup the user has. To make this work, we apply the expected shelf
 901          * filters for decoding UHJ2 to quad (only needs LF scaling), and act
 902          * as if those 4 quad channels are encoded right back onto first-order
 903          * B-Format, which then upsamples to higher order as normal (only needs
 904          * HF scaling).
 905          *
 906          * This isn't perfect, but without an entirely separate and limited
 907          * UHJ2 path, it's better than nothing.
 908          */
 909         if(mFmtChannels == FmtUHJ2)
 910         {
 911             mChans[0].mAmbiLFScale = UhjDecoder::sWLFScale;
 912             mChans[1].mAmbiLFScale = UhjDecoder::sXYLFScale;
 913             mChans[2].mAmbiLFScale = UhjDecoder::sXYLFScale;
 914         }
 915         mFlags.set(VoiceIsAmbisonic);
 916     }
 917     else if(mFmtChannels == FmtUHJ2 && !device->mUhjEncoder)
 918     {
 919         /* 2-channel UHJ with first-order output also needs the shelf filter
 920          * correction applied, except with UHJ output (UHJ2->B-Format->UHJ2 is
 921          * identity, so don't mess with it).
 922          */
 923         const BandSplitter splitter{device->mXOverFreq / static_cast<float>(device->Frequency)};
 924         for(auto &chandata : mChans)
 925         {
 926             chandata.mAmbiHFScale = 1.0f;
 927             chandata.mAmbiLFScale = 1.0f;
 928             chandata.mAmbiSplitter = splitter;
 929             chandata.mDryParams = DirectParams{};
 930             chandata.mDryParams.NFCtrlFilter = device->mNFCtrlFilter;
 931             std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
 932         }
 933         mChans[0].mAmbiLFScale = UhjDecoder::sWLFScale;
 934         mChans[1].mAmbiLFScale = UhjDecoder::sXYLFScale;
 935         mChans[2].mAmbiLFScale = UhjDecoder::sXYLFScale;
 936         mFlags.set(VoiceIsAmbisonic);
 937     }
 938     else
 939     {
 940         for(auto &chandata : mChans)
 941         {
 942             chandata.mDryParams = DirectParams{};
 943             chandata.mDryParams.NFCtrlFilter = device->mNFCtrlFilter;
 944             std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
 945         }
 946         mFlags.reset(VoiceIsAmbisonic);
 947     }
 948 }