core/voice.cpp

   1
   2 #include "config.h"
   3
   4 #include "voice.h"
   5
   6 #include <algorithm>
   7 #include <array>
   8 #include <atomic>
   9 #include <cassert>
  10 #include <cstdint>
  11 #include <iterator>
  12 #include <memory>
  13 #include <new>
  14 #include <stdlib.h>
  15 #include <utility>
  16 #include <vector>
  17
  18 #include "albyte.h"
  19 #include "alnumeric.h"
  20 #include "aloptional.h"
  21 #include "alspan.h"
  22 #include "alstring.h"
  23 #include "ambidefs.h"
  24 #include "async_event.h"
  25 #include "buffer_storage.h"
  26 #include "context.h"
  27 #include "cpu_caps.h"
  28 #include "devformat.h"
  29 #include "device.h"
  30 #include "filters/biquad.h"
  31 #include "filters/nfc.h"
  32 #include "filters/splitter.h"
  33 #include "fmt_traits.h"
  34 #include "logging.h"
  35 #include "mixer.h"
  36 #include "mixer/defs.h"
  37 #include "mixer/hrtfdefs.h"
  38 #include "opthelpers.h"
  39 #include "resampler_limits.h"
  40 #include "ringbuffer.h"
  41 #include "vector.h"
  42 #include "voice_change.h"
  43
  44 struct CTag;
  45 #ifdef HAVE_SSE
  46 struct SSETag;
  47 #endif
  48 #ifdef HAVE_NEON
  49 struct NEONTag;
  50 #endif
  51 struct CopyTag;
  52
  53
  54 static_assert(!(sizeof(DeviceBase::MixerBufferLine)&15),
  55     "DeviceBase::MixerBufferLine must be a multiple of 16 bytes");
  56 static_assert(!(MaxResamplerEdge&3), "MaxResamplerEdge is not a multiple of 4");
  57
  58 Resampler ResamplerDefault{Resampler::Linear};
  59
  60 namespace {
  61
  62 using uint = unsigned int;
  63 using namespace std::chrono;
  64
  65 using HrtfMixerFunc = void(*)(const float *InSamples, float2 *AccumSamples, const uint IrSize,
  66     const MixHrtfFilter *hrtfparams, const size_t BufferSize);
  67 using HrtfMixerBlendFunc = void(*)(const float *InSamples, float2 *AccumSamples,
  68     const uint IrSize, const HrtfFilter *oldparams, const MixHrtfFilter *newparams,
  69     const size_t BufferSize);
  70
  71 HrtfMixerFunc MixHrtfSamples{MixHrtf_<CTag>};
  72 HrtfMixerBlendFunc MixHrtfBlendSamples{MixHrtfBlend_<CTag>};
  73
  74 inline MixerOutFunc SelectMixer()
  75 {
  76 #ifdef HAVE_NEON
  77     if((CPUCapFlags&CPU_CAP_NEON))
  78         return Mix_<NEONTag>;
  79 #endif
  80 #ifdef HAVE_SSE
  81     if((CPUCapFlags&CPU_CAP_SSE))
  82         return Mix_<SSETag>;
  83 #endif
  84     return Mix_<CTag>;
  85 }
  86
  87 inline MixerOneFunc SelectMixerOne()
  88 {
  89 #ifdef HAVE_NEON
  90     if((CPUCapFlags&CPU_CAP_NEON))
  91         return Mix_<NEONTag>;
  92 #endif
  93 #ifdef HAVE_SSE
  94     if((CPUCapFlags&CPU_CAP_SSE))
  95         return Mix_<SSETag>;
  96 #endif
  97     return Mix_<CTag>;
  98 }
  99
 100 inline HrtfMixerFunc SelectHrtfMixer()
 101 {
 102 #ifdef HAVE_NEON
 103     if((CPUCapFlags&CPU_CAP_NEON))
 104         return MixHrtf_<NEONTag>;
 105 #endif
 106 #ifdef HAVE_SSE
 107     if((CPUCapFlags&CPU_CAP_SSE))
 108         return MixHrtf_<SSETag>;
 109 #endif
 110     return MixHrtf_<CTag>;
 111 }
 112
 113 inline HrtfMixerBlendFunc SelectHrtfBlendMixer()
 114 {
 115 #ifdef HAVE_NEON
 116     if((CPUCapFlags&CPU_CAP_NEON))
 117         return MixHrtfBlend_<NEONTag>;
 118 #endif
 119 #ifdef HAVE_SSE
 120     if((CPUCapFlags&CPU_CAP_SSE))
 121         return MixHrtfBlend_<SSETag>;
 122 #endif
 123     return MixHrtfBlend_<CTag>;
 124 }
 125
 126 } // namespace
 127
 128 void Voice::InitMixer(al::optional<std::string> resampler)
 129 {
 130     if(resampler)
 131     {
 132         struct ResamplerEntry {
 133             const char name[16];
 134             const Resampler resampler;
 135         };
 136         constexpr ResamplerEntry ResamplerList[]{
 137             { "none", Resampler::Point },
 138             { "point", Resampler::Point },
 139             { "linear", Resampler::Linear },
 140             { "cubic", Resampler::Cubic },
 141             { "bsinc12", Resampler::BSinc12 },
 142             { "fast_bsinc12", Resampler::FastBSinc12 },
 143             { "bsinc24", Resampler::BSinc24 },
 144             { "fast_bsinc24", Resampler::FastBSinc24 },
 145         };
 146
 147         const char *str{resampler->c_str()};
 148         if(al::strcasecmp(str, "bsinc") == 0)
 149         {
 150             WARN("Resampler option \"%s\" is deprecated, using bsinc12\n", str);
 151             str = "bsinc12";
 152         }
 153         else if(al::strcasecmp(str, "sinc4") == 0 || al::strcasecmp(str, "sinc8") == 0)
 154         {
 155             WARN("Resampler option \"%s\" is deprecated, using cubic\n", str);
 156             str = "cubic";
 157         }
 158
 159         auto iter = std::find_if(std::begin(ResamplerList), std::end(ResamplerList),
 160             [str](const ResamplerEntry &entry) -> bool
 161             { return al::strcasecmp(str, entry.name) == 0; });
 162         if(iter == std::end(ResamplerList))
 163             ERR("Invalid resampler: %s\n", str);
 164         else
 165             ResamplerDefault = iter->resampler;
 166     }
 167
 168     MixSamplesOut = SelectMixer();
 169     MixSamplesOne = SelectMixerOne();
 170     MixHrtfBlendSamples = SelectHrtfBlendMixer();
 171     MixHrtfSamples = SelectHrtfMixer();
 172 }
 173
 174
 175 namespace {
 176
 177 void SendSourceStoppedEvent(ContextBase *context, uint id)
 178 {
 179     RingBuffer *ring{context->mAsyncEvents.get()};
 180     auto evt_vec = ring->getWriteVector();
 181     if(evt_vec.first.len < 1) return;
 182
 183     AsyncEvent *evt{al::construct_at(reinterpret_cast<AsyncEvent*>(evt_vec.first.buf),
 184         AsyncEvent::SourceStateChange)};
 185     evt->u.srcstate.id = id;
 186     evt->u.srcstate.state = AsyncEvent::SrcState::Stop;
 187
 188     ring->writeAdvance(1);
 189 }
 190
 191
 192 const float *DoFilters(BiquadFilter &lpfilter, BiquadFilter &hpfilter, float *dst,
 193     const al::span<const float> src, int type)
 194 {
 195     switch(type)
 196     {
 197     case AF_None:
 198         lpfilter.clear();
 199         hpfilter.clear();
 200         break;
 201
 202     case AF_LowPass:
 203         lpfilter.process(src, dst);
 204         hpfilter.clear();
 205         return dst;
 206     case AF_HighPass:
 207         lpfilter.clear();
 208         hpfilter.process(src, dst);
 209         return dst;
 210
 211     case AF_BandPass:
 212         DualBiquad{lpfilter, hpfilter}.process(src, dst);
 213         return dst;
 214     }
 215     return src.data();
 216 }
 217
 218
 219 template<FmtType Type>
 220 inline void LoadSamples(const al::span<float*> dstSamples, const size_t dstOffset,
 221     const al::byte *src, const size_t srcOffset, const FmtChannels srcChans, const size_t srcStep,
 222     const size_t samples) noexcept
 223 {
 224     constexpr size_t sampleSize{sizeof(typename al::FmtTypeTraits<Type>::Type)};
 225     auto s = src + srcOffset*srcStep*sampleSize;
 226     if(srcChans == FmtUHJ2 || srcChans == FmtSuperStereo)
 227     {
 228         al::LoadSampleArray<Type>(dstSamples[0]+dstOffset, s, srcStep, samples);
 229         al::LoadSampleArray<Type>(dstSamples[1]+dstOffset, s+sampleSize, srcStep, samples);
 230         std::fill_n(dstSamples[2]+dstOffset, samples, 0.0f);
 231     }
 232     else
 233     {
 234         for(auto *dst : dstSamples)
 235         {
 236             al::LoadSampleArray<Type>(dst+dstOffset, s, srcStep, samples);
 237             s += sampleSize;
 238         }
 239     }
 240 }
 241
 242 void LoadSamples(const al::span<float*> dstSamples, const size_t dstOffset, const al::byte *src,
 243     const size_t srcOffset, const FmtType srcType, const FmtChannels srcChans,
 244     const size_t srcStep, const size_t samples) noexcept
 245 {
 246 #define HANDLE_FMT(T) case T:                                                 \
 247     LoadSamples<T>(dstSamples, dstOffset, src, srcOffset, srcChans, srcStep,  \
 248         samples);                                                             \
 249     break
 250
 251     switch(srcType)
 252     {
 253     HANDLE_FMT(FmtUByte);
 254     HANDLE_FMT(FmtShort);
 255     HANDLE_FMT(FmtFloat);
 256     HANDLE_FMT(FmtDouble);
 257     HANDLE_FMT(FmtMulaw);
 258     HANDLE_FMT(FmtAlaw);
 259     }
 260 #undef HANDLE_FMT
 261 }
 262
 263 void LoadBufferStatic(VoiceBufferItem *buffer, VoiceBufferItem *&bufferLoopItem,
 264     const size_t dataPosInt, const FmtType sampleType, const FmtChannels sampleChannels,
 265     const size_t srcStep, size_t samplesLoaded, const size_t samplesToLoad,
 266     const al::span<float*> voiceSamples)
 267 {
 268     const size_t loopStart{buffer->mLoopStart};
 269     const size_t loopEnd{buffer->mLoopEnd};
 270
 271     /* If current pos is beyond the loop range, do not loop */
 272     if(!bufferLoopItem || dataPosInt >= loopEnd)
 273     {
 274         bufferLoopItem = nullptr;
 275
 276         /* Load what's left to play from the buffer */
 277         const size_t remaining{minz(samplesToLoad-samplesLoaded, buffer->mSampleLen-dataPosInt)};
 278         LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, dataPosInt, sampleType,
 279             sampleChannels, srcStep, remaining);
 280         samplesLoaded += remaining;
 281
 282         if(const size_t toFill{samplesToLoad - samplesLoaded})
 283         {
 284             for(auto *chanbuffer : voiceSamples)
 285             {
 286                 auto srcsamples = chanbuffer + samplesLoaded - 1;
 287                 std::fill_n(srcsamples + 1, toFill, *srcsamples);
 288             }
 289         }
 290     }
 291     else
 292     {
 293         ASSUME(loopEnd > loopStart);
 294
 295         /* Load what's left of this loop iteration */
 296         const size_t remaining{minz(samplesToLoad-samplesLoaded, loopEnd-dataPosInt)};
 297         LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, dataPosInt, sampleType,
 298             sampleChannels, srcStep, remaining);
 299         samplesLoaded += remaining;
 300
 301         /* Load repeats of the loop to fill the buffer. */
 302         const size_t loopSize{loopEnd - loopStart};
 303         while(const size_t toFill{minz(samplesToLoad - samplesLoaded, loopSize)})
 304         {
 305             LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, loopStart, sampleType,
 306                 sampleChannels, srcStep, toFill);
 307             samplesLoaded += toFill;
 308         }
 309     }
 310 }
 311
 312 void LoadBufferCallback(VoiceBufferItem *buffer, const size_t numCallbackSamples,
 313     const FmtType sampleType, const FmtChannels sampleChannels, const size_t srcStep,
 314     size_t samplesLoaded, const size_t samplesToLoad, const al::span<float*> voiceSamples)
 315 {
 316     /* Load what's left to play from the buffer */
 317     const size_t remaining{minz(samplesToLoad-samplesLoaded, numCallbackSamples)};
 318     LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, 0, sampleType, sampleChannels,
 319         srcStep, remaining);
 320     samplesLoaded += remaining;
 321
 322     if(const size_t toFill{samplesToLoad - samplesLoaded})
 323     {
 324         for(auto *chanbuffer : voiceSamples)
 325         {
 326             auto srcsamples = chanbuffer + remaining;
 327             std::fill_n(srcsamples, toFill, *(srcsamples-1));
 328         }
 329     }
 330 }
 331
 332 void LoadBufferQueue(VoiceBufferItem *buffer, VoiceBufferItem *bufferLoopItem,
 333     size_t dataPosInt, const FmtType sampleType, const FmtChannels sampleChannels,
 334     const size_t srcStep, size_t samplesLoaded, const size_t samplesToLoad,
 335     const al::span<float*> voiceSamples)
 336 {
 337     /* Crawl the buffer queue to fill in the temp buffer */
 338     while(buffer && samplesLoaded != samplesToLoad)
 339     {
 340         if(dataPosInt >= buffer->mSampleLen)
 341         {
 342             dataPosInt -= buffer->mSampleLen;
 343             buffer = buffer->mNext.load(std::memory_order_acquire);
 344             if(!buffer) buffer = bufferLoopItem;
 345             continue;
 346         }
 347
 348         const size_t remaining{minz(samplesToLoad-samplesLoaded, buffer->mSampleLen-dataPosInt)};
 349         LoadSamples(voiceSamples, samplesLoaded, buffer->mSamples, dataPosInt, sampleType,
 350             sampleChannels, srcStep, remaining);
 351
 352         samplesLoaded += remaining;
 353         if(samplesLoaded == samplesToLoad)
 354             break;
 355
 356         dataPosInt = 0;
 357         buffer = buffer->mNext.load(std::memory_order_acquire);
 358         if(!buffer) buffer = bufferLoopItem;
 359     }
 360     if(const size_t toFill{samplesToLoad - samplesLoaded})
 361     {
 362         for(auto *chanbuffer : voiceSamples)
 363         {
 364             auto srcsamples = chanbuffer + samplesLoaded;
 365             std::fill_n(srcsamples, toFill, *(srcsamples-1));
 366         }
 367     }
 368 }
 369
 370
 371 void DoHrtfMix(const float *samples, const uint DstBufferSize, DirectParams &parms,
 372     const float TargetGain, const uint Counter, uint OutPos, const bool IsPlaying,
 373     DeviceBase *Device)
 374 {
 375     const uint IrSize{Device->mIrSize};
 376     auto &HrtfSamples = Device->HrtfSourceData;
 377     auto &AccumSamples = Device->HrtfAccumData;
 378
 379     /* Copy the HRTF history and new input samples into a temp buffer. */
 380     auto src_iter = std::copy(parms.Hrtf.History.begin(), parms.Hrtf.History.end(),
 381         std::begin(HrtfSamples));
 382     std::copy_n(samples, DstBufferSize, src_iter);
 383     /* Copy the last used samples back into the history buffer for later. */
 384     if(IsPlaying) [[likely]]
 385         std::copy_n(std::begin(HrtfSamples) + DstBufferSize, parms.Hrtf.History.size(),
 386             parms.Hrtf.History.begin());
 387
 388     /* If fading and this is the first mixing pass, fade between the IRs. */
 389     uint fademix{0u};
 390     if(Counter && OutPos == 0)
 391     {
 392         fademix = minu(DstBufferSize, Counter);
 393
 394         float gain{TargetGain};
 395
 396         /* The new coefficients need to fade in completely since they're
 397          * replacing the old ones. To keep the gain fading consistent,
 398          * interpolate between the old and new target gains given how much of
 399          * the fade time this mix handles.
 400          */
 401         if(Counter > fademix)
 402         {
 403             const float a{static_cast<float>(fademix) / static_cast<float>(Counter)};
 404             gain = lerpf(parms.Hrtf.Old.Gain, TargetGain, a);
 405         }
 406
 407         MixHrtfFilter hrtfparams{
 408             parms.Hrtf.Target.Coeffs,
 409             parms.Hrtf.Target.Delay,
 410             0.0f, gain / static_cast<float>(fademix)};
 411         MixHrtfBlendSamples(HrtfSamples, AccumSamples+OutPos, IrSize, &parms.Hrtf.Old, &hrtfparams,
 412             fademix);
 413
 414         /* Update the old parameters with the result. */
 415         parms.Hrtf.Old = parms.Hrtf.Target;
 416         parms.Hrtf.Old.Gain = gain;
 417         OutPos += fademix;
 418     }
 419
 420     if(fademix < DstBufferSize)
 421     {
 422         const uint todo{DstBufferSize - fademix};
 423         float gain{TargetGain};
 424
 425         /* Interpolate the target gain if the gain fading lasts longer than
 426          * this mix.
 427          */
 428         if(Counter > DstBufferSize)
 429         {
 430             const float a{static_cast<float>(todo) / static_cast<float>(Counter-fademix)};
 431             gain = lerpf(parms.Hrtf.Old.Gain, TargetGain, a);
 432         }
 433
 434         MixHrtfFilter hrtfparams{
 435             parms.Hrtf.Target.Coeffs,
 436             parms.Hrtf.Target.Delay,
 437             parms.Hrtf.Old.Gain,
 438             (gain - parms.Hrtf.Old.Gain) / static_cast<float>(todo)};
 439         MixHrtfSamples(HrtfSamples+fademix, AccumSamples+OutPos, IrSize, &hrtfparams, todo);
 440
 441         /* Store the now-current gain for next time. */
 442         parms.Hrtf.Old.Gain = gain;
 443     }
 444 }
 445
 446 void DoNfcMix(const al::span<const float> samples, FloatBufferLine *OutBuffer, DirectParams &parms,
 447     const float *TargetGains, const uint Counter, const uint OutPos, DeviceBase *Device)
 448 {
 449     using FilterProc = void (NfcFilter::*)(const al::span<const float>, float*);
 450     static constexpr FilterProc NfcProcess[MaxAmbiOrder+1]{
 451         nullptr, &NfcFilter::process1, &NfcFilter::process2, &NfcFilter::process3};
 452
 453     float *CurrentGains{parms.Gains.Current.data()};
 454     MixSamples(samples, {OutBuffer, 1u}, CurrentGains, TargetGains, Counter, OutPos);
 455     ++OutBuffer;
 456     ++CurrentGains;
 457     ++TargetGains;
 458
 459     const al::span<float> nfcsamples{Device->NfcSampleData, samples.size()};
 460     size_t order{1};
 461     while(const size_t chancount{Device->NumChannelsPerOrder[order]})
 462     {
 463         (parms.NFCtrlFilter.*NfcProcess[order])(samples, nfcsamples.data());
 464         MixSamples(nfcsamples, {OutBuffer, chancount}, CurrentGains, TargetGains, Counter, OutPos);
 465         OutBuffer += chancount;
 466         CurrentGains += chancount;
 467         TargetGains += chancount;
 468         if(++order == MaxAmbiOrder+1)
 469             break;
 470     }
 471 }
 472
 473 } // namespace
 474
 475 void Voice::mix(const State vstate, ContextBase *Context, const nanoseconds deviceTime,
 476     const uint SamplesToDo)
 477 {
 478     static constexpr std::array<float,MAX_OUTPUT_CHANNELS> SilentTarget{};
 479
 480     ASSUME(SamplesToDo > 0);
 481
 482     DeviceBase *Device{Context->mDevice};
 483     const uint NumSends{Device->NumAuxSends};
 484
 485     /* Get voice info */
 486     int DataPosInt{mPosition.load(std::memory_order_relaxed)};
 487     uint DataPosFrac{mPositionFrac.load(std::memory_order_relaxed)};
 488     VoiceBufferItem *BufferListItem{mCurrentBuffer.load(std::memory_order_relaxed)};
 489     VoiceBufferItem *BufferLoopItem{mLoopBuffer.load(std::memory_order_relaxed)};
 490     const uint increment{mStep};
 491     if(increment < 1) [[unlikely]]
 492     {
 493         /* If the voice is supposed to be stopping but can't be mixed, just
 494          * stop it before bailing.
 495          */
 496         if(vstate == Stopping)
 497             mPlayState.store(Stopped, std::memory_order_release);
 498         return;
 499     }
 500
 501     uint Counter{mFlags.test(VoiceIsFading) ? minu(SamplesToDo, 64u) : 0u};
 502     uint OutPos{0u};
 503
 504     /* Check if we're doing a delayed start, and we start in this update. */
 505     if(mStartTime > deviceTime)
 506     {
 507         /* If the start time is too far ahead, don't bother. */
 508         auto diff = mStartTime - deviceTime;
 509         if(diff >= seconds{1})
 510             return;
 511
 512         /* Get the number of samples ahead of the current time that output
 513          * should start at. Skip this update if it's beyond the output sample
 514          * count.
 515          *
 516          * Round the start position to a multiple of 4, which some mixers want.
 517          * This makes the start time accurate to 4 samples. This could be made
 518          * sample-accurate by forcing non-SIMD functions on the first run.
 519          */
 520         seconds::rep sampleOffset{duration_cast<seconds>(diff * Device->Frequency).count()};
 521         sampleOffset = (sampleOffset+2) & ~seconds::rep{3};
 522         if(sampleOffset >= SamplesToDo)
 523             return;
 524
 525         OutPos = static_cast<uint>(sampleOffset);
 526     }
 527
 528     if(!Counter)
 529     {
 530         /* No fading, just overwrite the old/current params. */
 531         for(auto &chandata : mChans)
 532         {
 533             {
 534                 DirectParams &parms = chandata.mDryParams;
 535                 if(!mFlags.test(VoiceHasHrtf))
 536                     parms.Gains.Current = parms.Gains.Target;
 537                 else
 538                     parms.Hrtf.Old = parms.Hrtf.Target;
 539             }
 540             for(uint send{0};send < NumSends;++send)
 541             {
 542                 if(mSend[send].Buffer.empty())
 543                     continue;
 544
 545                 SendParams &parms = chandata.mWetParams[send];
 546                 parms.Gains.Current = parms.Gains.Target;
 547             }
 548         }
 549     }
 550
 551     std::array<float*,DeviceBase::MixerChannelsMax> SamplePointers;
 552     const al::span<float*> MixingSamples{SamplePointers.data(), mChans.size()};
 553     auto offset_bufferline = [](DeviceBase::MixerBufferLine &bufline) noexcept -> float*
 554     { return bufline.data() + MaxResamplerEdge; };
 555     std::transform(Device->mSampleData.end() - mChans.size(), Device->mSampleData.end(),
 556         MixingSamples.begin(), offset_bufferline);
 557
 558     const ResamplerFunc Resample{(increment == MixerFracOne && DataPosFrac == 0) ?
 559         Resample_<CopyTag,CTag> : mResampler};
 560     const uint PostPadding{MaxResamplerEdge + mDecoderPadding};
 561     uint buffers_done{0u};
 562     do {
 563         /* Figure out how many buffer samples will be needed */
 564         uint DstBufferSize{SamplesToDo - OutPos};
 565         uint SrcBufferSize;
 566
 567         if(increment <= MixerFracOne)
 568         {
 569             /* Calculate the last written dst sample pos. */
 570             uint64_t DataSize64{DstBufferSize - 1};
 571             /* Calculate the last read src sample pos. */
 572             DataSize64 = (DataSize64*increment + DataPosFrac) >> MixerFracBits;
 573             /* +1 to get the src sample count, include padding. */
 574             DataSize64 += 1 + PostPadding;
 575
 576             /* Result is guaranteed to be <= BufferLineSize+PostPadding since
 577              * we won't use more src samples than dst samples+padding.
 578              */
 579             SrcBufferSize = static_cast<uint>(DataSize64);
 580         }
 581         else
 582         {
 583             uint64_t DataSize64{DstBufferSize};
 584             /* Calculate the end src sample pos, include padding. */
 585             DataSize64 = (DataSize64*increment + DataPosFrac) >> MixerFracBits;
 586             DataSize64 += PostPadding;
 587
 588             if(DataSize64 <= DeviceBase::MixerLineSize - MaxResamplerEdge)
 589                 SrcBufferSize = static_cast<uint>(DataSize64);
 590             else
 591             {
 592                 /* If the source size got saturated, we can't fill the desired
 593                  * dst size. Figure out how many samples we can actually mix.
 594                  */
 595                 SrcBufferSize = DeviceBase::MixerLineSize - MaxResamplerEdge;
 596
 597                 DataSize64 = SrcBufferSize - PostPadding;
 598                 DataSize64 = ((DataSize64<<MixerFracBits) - DataPosFrac) / increment;
 599                 if(DataSize64 < DstBufferSize)
 600                 {
 601                     /* Some mixers require being 16-byte aligned, so also limit
 602                      * to a multiple of 4 samples to maintain alignment.
 603                      */
 604                     DstBufferSize = static_cast<uint>(DataSize64) & ~3u;
 605                     /* If the voice is stopping, only one mixing iteration will
 606                      * be done, so ensure it fades out completely this mix.
 607                      */
 608                     if(vstate == Stopping) [[unlikely]]
 609                         Counter = std::min(Counter, DstBufferSize);
 610                 }
 611                 ASSUME(DstBufferSize > 0);
 612             }
 613         }
 614
 615         float **voiceSamples{};
 616         if(!BufferListItem) [[unlikely]]
 617         {
 618             const size_t srcOffset{(increment*DstBufferSize + DataPosFrac)>>MixerFracBits};
 619             auto prevSamples = mPrevSamples.data();
 620             SrcBufferSize = SrcBufferSize - PostPadding + MaxResamplerEdge;
 621             for(auto *chanbuffer : MixingSamples)
 622             {
 623                 auto srcend = std::copy_n(prevSamples->data(), MaxResamplerPadding,
 624                     chanbuffer-MaxResamplerEdge);
 625
 626                 /* When loading from a voice that ended prematurely, only take
 627                  * the samples that get closest to 0 amplitude. This helps
 628                  * certain sounds fade out better.
 629                  */
 630                 auto abs_lt = [](const float lhs, const float rhs) noexcept -> bool
 631                 { return std::abs(lhs) < std::abs(rhs); };
 632                 auto srciter = std::min_element(chanbuffer, srcend, abs_lt);
 633
 634                 std::fill(srciter+1, chanbuffer + SrcBufferSize, *srciter);
 635
 636                 std::copy_n(chanbuffer-MaxResamplerEdge+srcOffset, prevSamples->size(),
 637                     prevSamples->data());
 638                 ++prevSamples;
 639             }
 640         }
 641         else
 642         {
 643             auto prevSamples = mPrevSamples.data();
 644             for(auto *chanbuffer : MixingSamples)
 645             {
 646                 std::copy_n(prevSamples->data(), MaxResamplerEdge, chanbuffer-MaxResamplerEdge);
 647                 ++prevSamples;
 648             }
 649
 650             size_t samplesLoaded{0};
 651             if(DataPosInt < 0) [[unlikely]]
 652             {
 653                 if(static_cast<uint>(-DataPosInt) >= SrcBufferSize)
 654                     goto skip_mix;
 655
 656                 samplesLoaded = static_cast<uint>(-DataPosInt);
 657                 for(auto *chanbuffer : MixingSamples)
 658                     std::fill_n(chanbuffer, samplesLoaded, 0.0f);
 659             }
 660             const uint DataPosUInt{static_cast<uint>(maxi(DataPosInt, 0))};
 661
 662             if(mFlags.test(VoiceIsStatic))
 663                 LoadBufferStatic(BufferListItem, BufferLoopItem, DataPosUInt, mFmtType,
 664                     mFmtChannels, mFrameStep, samplesLoaded, SrcBufferSize, MixingSamples);
 665             else if(mFlags.test(VoiceIsCallback))
 666             {
 667                 const size_t remaining{SrcBufferSize - samplesLoaded};
 668                 if(!mFlags.test(VoiceCallbackStopped) && remaining > mNumCallbackSamples)
 669                 {
 670                     const size_t byteOffset{mNumCallbackSamples*mFrameSize};
 671                     const size_t needBytes{remaining*mFrameSize - byteOffset};
 672
 673                     const int gotBytes{BufferListItem->mCallback(BufferListItem->mUserData,
 674                         &BufferListItem->mSamples[byteOffset], static_cast<int>(needBytes))};
 675                     if(gotBytes < 0)
 676                         mFlags.set(VoiceCallbackStopped);
 677                     else if(static_cast<uint>(gotBytes) < needBytes)
 678                     {
 679                         mFlags.set(VoiceCallbackStopped);
 680                         mNumCallbackSamples += static_cast<uint>(gotBytes) / mFrameSize;
 681                     }
 682                     else
 683                         mNumCallbackSamples = static_cast<uint>(remaining);
 684                 }
 685                 LoadBufferCallback(BufferListItem, mNumCallbackSamples, mFmtType, mFmtChannels,
 686                     mFrameStep, samplesLoaded, SrcBufferSize, MixingSamples);
 687             }
 688             else
 689                 LoadBufferQueue(BufferListItem, BufferLoopItem, DataPosUInt, mFmtType, mFmtChannels,
 690                     mFrameStep, samplesLoaded, SrcBufferSize, MixingSamples);
 691
 692             const size_t srcOffset{(increment*DstBufferSize + DataPosFrac)>>MixerFracBits};
 693             if(mDecoder)
 694             {
 695                 SrcBufferSize = SrcBufferSize - PostPadding + MaxResamplerEdge;
 696                 mDecoder->decode(MixingSamples, SrcBufferSize,
 697                     (vstate == Playing) ? srcOffset : 0);
 698             }
 699
 700             /* Store the last source samples used for next time. */
 701             if(vstate == Playing) [[likely]]
 702             {
 703                 prevSamples = mPrevSamples.data();
 704                 for(auto *chanbuffer : MixingSamples)
 705                 {
 706                     std::copy_n(chanbuffer-MaxResamplerEdge+srcOffset, prevSamples->size(),
 707                         prevSamples->data());
 708                     ++prevSamples;
 709                 }
 710             }
 711         }
 712
 713         voiceSamples = MixingSamples.begin();
 714         for(auto &chandata : mChans)
 715         {
 716             /* Resample, then apply ambisonic upsampling as needed. */
 717             float *ResampledData{Resample(&mResampleState, *voiceSamples, DataPosFrac, increment,
 718                 {Device->ResampledData, DstBufferSize})};
 719             ++voiceSamples;
 720
 721             if(mFlags.test(VoiceIsAmbisonic))
 722                 chandata.mAmbiSplitter.processScale({ResampledData, DstBufferSize},
 723                     chandata.mAmbiHFScale, chandata.mAmbiLFScale);
 724
 725             /* Now filter and mix to the appropriate outputs. */
 726             const al::span<float,BufferLineSize> FilterBuf{Device->FilteredData};
 727             {
 728                 DirectParams &parms = chandata.mDryParams;
 729                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf.data(),
 730                     {ResampledData, DstBufferSize}, mDirect.FilterType)};
 731
 732                 if(mFlags.test(VoiceHasHrtf))
 733                 {
 734                     const float TargetGain{parms.Hrtf.Target.Gain * (vstate == Playing)};
 735                     DoHrtfMix(samples, DstBufferSize, parms, TargetGain, Counter, OutPos,
 736                         (vstate == Playing), Device);
 737                 }
 738                 else
 739                 {
 740                     const float *TargetGains{(vstate == Playing) ? parms.Gains.Target.data()
 741                         : SilentTarget.data()};
 742                     if(mFlags.test(VoiceHasNfc))
 743                         DoNfcMix({samples, DstBufferSize}, mDirect.Buffer.data(), parms,
 744                             TargetGains, Counter, OutPos, Device);
 745                     else
 746                         MixSamples({samples, DstBufferSize}, mDirect.Buffer,
 747                             parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 748                 }
 749             }
 750
 751             for(uint send{0};send < NumSends;++send)
 752             {
 753                 if(mSend[send].Buffer.empty())
 754                     continue;
 755
 756                 SendParams &parms = chandata.mWetParams[send];
 757                 const float *samples{DoFilters(parms.LowPass, parms.HighPass, FilterBuf.data(),
 758                     {ResampledData, DstBufferSize}, mSend[send].FilterType)};
 759
 760                 const float *TargetGains{(vstate == Playing) ? parms.Gains.Target.data()
 761                     : SilentTarget.data()};
 762                 MixSamples({samples, DstBufferSize}, mSend[send].Buffer,
 763                     parms.Gains.Current.data(), TargetGains, Counter, OutPos);
 764             }
 765         }
 766     skip_mix:
 767         /* If the voice is stopping, we're now done. */
 768         if(vstate == Stopping) [[unlikely]]
 769             break;
 770
 771         /* Update positions */
 772         DataPosFrac += increment*DstBufferSize;
 773         const uint SrcSamplesDone{DataPosFrac>>MixerFracBits};
 774         DataPosInt  += SrcSamplesDone;
 775         DataPosFrac &= MixerFracMask;
 776
 777         OutPos += DstBufferSize;
 778         Counter = maxu(DstBufferSize, Counter) - DstBufferSize;
 779
 780         /* Do nothing extra when there's no buffers, or if the voice position
 781          * is still negative.
 782          */
 783         if(!BufferListItem || DataPosInt < 0) [[unlikely]]
 784             continue;
 785
 786         if(mFlags.test(VoiceIsStatic))
 787         {
 788             if(BufferLoopItem)
 789             {
 790                 /* Handle looping static source */
 791                 const uint LoopStart{BufferListItem->mLoopStart};
 792                 const uint LoopEnd{BufferListItem->mLoopEnd};
 793                 uint DataPosUInt{static_cast<uint>(DataPosInt)};
 794                 if(DataPosUInt >= LoopEnd)
 795                 {
 796                     assert(LoopEnd > LoopStart);
 797                     DataPosUInt = ((DataPosUInt-LoopStart)%(LoopEnd-LoopStart)) + LoopStart;
 798                     DataPosInt = static_cast<int>(DataPosUInt);
 799                 }
 800             }
 801             else
 802             {
 803                 /* Handle non-looping static source */
 804                 if(static_cast<uint>(DataPosInt) >= BufferListItem->mSampleLen)
 805                 {
 806                     BufferListItem = nullptr;
 807                     break;
 808                 }
 809             }
 810         }
 811         else if(mFlags.test(VoiceIsCallback))
 812         {
 813             /* Handle callback buffer source */
 814             if(SrcSamplesDone < mNumCallbackSamples)
 815             {
 816                 const size_t byteOffset{SrcSamplesDone*mFrameSize};
 817                 const size_t byteEnd{mNumCallbackSamples*mFrameSize};
 818                 al::byte *data{BufferListItem->mSamples};
 819                 std::copy(data+byteOffset, data+byteEnd, data);
 820                 mNumCallbackSamples -= SrcSamplesDone;
 821             }
 822             else
 823             {
 824                 BufferListItem = nullptr;
 825                 mNumCallbackSamples = 0;
 826             }
 827         }
 828         else
 829         {
 830             /* Handle streaming source */
 831             do {
 832                 if(BufferListItem->mSampleLen > static_cast<uint>(DataPosInt))
 833                     break;
 834
 835                 DataPosInt -= BufferListItem->mSampleLen;
 836
 837                 ++buffers_done;
 838                 BufferListItem = BufferListItem->mNext.load(std::memory_order_relaxed);
 839                 if(!BufferListItem) BufferListItem = BufferLoopItem;
 840             } while(BufferListItem);
 841         }
 842     } while(OutPos < SamplesToDo);
 843
 844     mFlags.set(VoiceIsFading);
 845
 846     /* Don't update positions and buffers if we were stopping. */
 847     if(vstate == Stopping) [[unlikely]]
 848     {
 849         mPlayState.store(Stopped, std::memory_order_release);
 850         return;
 851     }
 852
 853     /* Capture the source ID in case it's reset for stopping. */
 854     const uint SourceID{mSourceID.load(std::memory_order_relaxed)};
 855
 856     /* Update voice info */
 857     mPosition.store(DataPosInt, std::memory_order_relaxed);
 858     mPositionFrac.store(DataPosFrac, std::memory_order_relaxed);
 859     mCurrentBuffer.store(BufferListItem, std::memory_order_relaxed);
 860     if(!BufferListItem)
 861     {
 862         mLoopBuffer.store(nullptr, std::memory_order_relaxed);
 863         mSourceID.store(0u, std::memory_order_relaxed);
 864     }
 865     std::atomic_thread_fence(std::memory_order_release);
 866
 867     /* Send any events now, after the position/buffer info was updated. */
 868     const auto enabledevt = Context->mEnabledEvts.load(std::memory_order_acquire);
 869     if(buffers_done > 0 && enabledevt.test(AsyncEvent::BufferCompleted))
 870     {
 871         RingBuffer *ring{Context->mAsyncEvents.get()};
 872         auto evt_vec = ring->getWriteVector();
 873         if(evt_vec.first.len > 0)
 874         {
 875             AsyncEvent *evt{al::construct_at(reinterpret_cast<AsyncEvent*>(evt_vec.first.buf),
 876                 AsyncEvent::BufferCompleted)};
 877             evt->u.bufcomp.id = SourceID;
 878             evt->u.bufcomp.count = buffers_done;
 879             ring->writeAdvance(1);
 880         }
 881     }
 882
 883     if(!BufferListItem)
 884     {
 885         /* If the voice just ended, set it to Stopping so the next render
 886          * ensures any residual noise fades to 0 amplitude.
 887          */
 888         mPlayState.store(Stopping, std::memory_order_release);
 889         if(enabledevt.test(AsyncEvent::SourceStateChange))
 890             SendSourceStoppedEvent(Context, SourceID);
 891     }
 892 }
 893
 894 void Voice::prepare(DeviceBase *device)
 895 {
 896     /* Even if storing really high order ambisonics, we only mix channels for
 897      * orders up to the device order. The rest are simply dropped.
 898      */
 899     uint num_channels{(mFmtChannels == FmtUHJ2 || mFmtChannels == FmtSuperStereo) ? 3 :
 900         ChannelsFromFmt(mFmtChannels, minu(mAmbiOrder, device->mAmbiOrder))};
 901     if(num_channels > device->mSampleData.size()) [[unlikely]]
 902     {
 903         ERR("Unexpected channel count: %u (limit: %zu, %d:%d)\n", num_channels,
 904             device->mSampleData.size(), mFmtChannels, mAmbiOrder);
 905         num_channels = static_cast<uint>(device->mSampleData.size());
 906     }
 907     if(mChans.capacity() > 2 && num_channels < mChans.capacity())
 908     {
 909         decltype(mChans){}.swap(mChans);
 910         decltype(mPrevSamples){}.swap(mPrevSamples);
 911     }
 912     mChans.reserve(maxu(2, num_channels));
 913     mChans.resize(num_channels);
 914     mPrevSamples.reserve(maxu(2, num_channels));
 915     mPrevSamples.resize(num_channels);
 916
 917     mDecoder = nullptr;
 918     mDecoderPadding = 0;
 919     if(mFmtChannels == FmtSuperStereo)
 920     {
 921         switch(UhjDecodeQuality)
 922         {
 923         case UhjQualityType::IIR:
 924             mDecoder = std::make_unique<UhjStereoDecoderIIR>();
 925             mDecoderPadding = UhjStereoDecoderIIR::sInputPadding;
 926             break;
 927         case UhjQualityType::FIR256:
 928             mDecoder = std::make_unique<UhjStereoDecoder<UhjLength256>>();
 929             mDecoderPadding = UhjStereoDecoder<UhjLength256>::sInputPadding;
 930             break;
 931         case UhjQualityType::FIR512:
 932             mDecoder = std::make_unique<UhjStereoDecoder<UhjLength512>>();
 933             mDecoderPadding = UhjStereoDecoder<UhjLength512>::sInputPadding;
 934             break;
 935         }
 936     }
 937     else if(IsUHJ(mFmtChannels))
 938     {
 939         switch(UhjDecodeQuality)
 940         {
 941         case UhjQualityType::IIR:
 942             mDecoder = std::make_unique<UhjDecoderIIR>();
 943             mDecoderPadding = UhjDecoderIIR::sInputPadding;
 944             break;
 945         case UhjQualityType::FIR256:
 946             mDecoder = std::make_unique<UhjDecoder<UhjLength256>>();
 947             mDecoderPadding = UhjDecoder<UhjLength256>::sInputPadding;
 948             break;
 949         case UhjQualityType::FIR512:
 950             mDecoder = std::make_unique<UhjDecoder<UhjLength512>>();
 951             mDecoderPadding = UhjDecoder<UhjLength512>::sInputPadding;
 952             break;
 953         }
 954     }
 955
 956     /* Clear the stepping value explicitly so the mixer knows not to mix this
 957      * until the update gets applied.
 958      */
 959     mStep = 0;
 960
 961     /* Make sure the sample history is cleared. */
 962     std::fill(mPrevSamples.begin(), mPrevSamples.end(), HistoryLine{});
 963
 964     if(mFmtChannels == FmtUHJ2 && !device->mUhjEncoder)
 965     {
 966         /* 2-channel UHJ needs different shelf filters. However, we can't just
 967          * use different shelf filters after mixing it, given any old speaker
 968          * setup the user has. To make this work, we apply the expected shelf
 969          * filters for decoding UHJ2 to quad (only needs LF scaling), and act
 970          * as if those 4 quad channels are encoded right back into B-Format.
 971          *
 972          * This isn't perfect, but without an entirely separate and limited
 973          * UHJ2 path, it's better than nothing.
 974          *
 975          * Note this isn't needed with UHJ output (UHJ2->B-Format->UHJ2 is
 976          * identity, so don't mess with it).
 977          */
 978         const BandSplitter splitter{device->mXOverFreq / static_cast<float>(device->Frequency)};
 979         for(auto &chandata : mChans)
 980         {
 981             chandata.mAmbiHFScale = 1.0f;
 982             chandata.mAmbiLFScale = 1.0f;
 983             chandata.mAmbiSplitter = splitter;
 984             chandata.mDryParams = DirectParams{};
 985             chandata.mDryParams.NFCtrlFilter = device->mNFCtrlFilter;
 986             std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
 987         }
 988         mChans[0].mAmbiLFScale = DecoderBase::sWLFScale;
 989         mChans[1].mAmbiLFScale = DecoderBase::sXYLFScale;
 990         mChans[2].mAmbiLFScale = DecoderBase::sXYLFScale;
 991         mFlags.set(VoiceIsAmbisonic);
 992     }
 993     /* Don't need to set the VoiceIsAmbisonic flag if the device is not higher
 994      * order than the voice. No HF scaling is necessary to mix it.
 995      */
 996     else if(mAmbiOrder && device->mAmbiOrder > mAmbiOrder)
 997     {
 998         const uint8_t *OrderFromChan{Is2DAmbisonic(mFmtChannels) ?
 999             AmbiIndex::OrderFrom2DChannel().data() : AmbiIndex::OrderFromChannel().data()};
1000         const auto scales = AmbiScale::GetHFOrderScales(mAmbiOrder, device->mAmbiOrder,
1001             device->m2DMixing);
1002
1003         const BandSplitter splitter{device->mXOverFreq / static_cast<float>(device->Frequency)};
1004         for(auto &chandata : mChans)
1005         {
1006             chandata.mAmbiHFScale = scales[*(OrderFromChan++)];
1007             chandata.mAmbiLFScale = 1.0f;
1008             chandata.mAmbiSplitter = splitter;
1009             chandata.mDryParams = DirectParams{};
1010             chandata.mDryParams.NFCtrlFilter = device->mNFCtrlFilter;
1011             std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
1012         }
1013         mFlags.set(VoiceIsAmbisonic);
1014     }
1015     else
1016     {
1017         for(auto &chandata : mChans)
1018         {
1019             chandata.mDryParams = DirectParams{};
1020             chandata.mDryParams.NFCtrlFilter = device->mNFCtrlFilter;
1021             std::fill_n(chandata.mWetParams.begin(), device->NumAuxSends, SendParams{});
1022         }
1023         mFlags.reset(VoiceIsAmbisonic);
1024     }
1025 }