examples/alffplay.cpp

   1 /*
   2  * An example showing how to play a stream sync'd to video, using ffmpeg.
   3  *
   4  * Requires C++14.
   5  */
   6
   7 #include <condition_variable>
   8 #include <functional>
   9 #include <algorithm>
  10 #include <iostream>
  11 #include <utility>
  12 #include <iomanip>
  13 #include <cstdint>
  14 #include <cstring>
  15 #include <cstdlib>
  16 #include <atomic>
  17 #include <cerrno>
  18 #include <chrono>
  19 #include <cstdio>
  20 #include <future>
  21 #include <memory>
  22 #include <string>
  23 #include <thread>
  24 #include <vector>
  25 #include <array>
  26 #include <cmath>
  27 #include <deque>
  28 #include <mutex>
  29 #include <ratio>
  30
  31 #ifdef __GNUC__
  32 _Pragma("GCC diagnostic push")
  33 _Pragma("GCC diagnostic ignored \"-Wconversion\"")
  34 _Pragma("GCC diagnostic ignored \"-Wold-style-cast\"")
  35 #endif
  36 extern "C" {
  37 #include "libavcodec/avcodec.h"
  38 #include "libavformat/avformat.h"
  39 #include "libavformat/avio.h"
  40 #include "libavformat/version.h"
  41 #include "libavutil/avutil.h"
  42 #include "libavutil/error.h"
  43 #include "libavutil/frame.h"
  44 #include "libavutil/mem.h"
  45 #include "libavutil/pixfmt.h"
  46 #include "libavutil/rational.h"
  47 #include "libavutil/samplefmt.h"
  48 #include "libavutil/time.h"
  49 #include "libavutil/version.h"
  50 #include "libavutil/channel_layout.h"
  51 #include "libswscale/swscale.h"
  52 #include "libswresample/swresample.h"
  53
  54 constexpr auto AVNoPtsValue = AV_NOPTS_VALUE;
  55 constexpr auto AVErrorEOF = AVERROR_EOF;
  56
  57 struct SwsContext;
  58 }
  59
  60 #include "SDL.h"
  61 #ifdef __GNUC__
  62 _Pragma("GCC diagnostic pop")
  63 #endif
  64
  65 #include "AL/alc.h"
  66 #include "AL/al.h"
  67 #include "AL/alext.h"
  68
  69 #include "common/alhelpers.h"
  70
  71
  72 namespace {
  73
  74 inline constexpr int64_t operator "" _i64(unsigned long long int n) noexcept { return static_cast<int64_t>(n); }
  75
  76 #ifndef M_PI
  77 #define M_PI (3.14159265358979323846)
  78 #endif
  79
  80 using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1_i64<<32)>>;
  81 using nanoseconds = std::chrono::nanoseconds;
  82 using microseconds = std::chrono::microseconds;
  83 using milliseconds = std::chrono::milliseconds;
  84 using seconds = std::chrono::seconds;
  85 using seconds_d64 = std::chrono::duration<double>;
  86 using std::chrono::duration_cast;
  87
  88 const std::string AppName{"alffplay"};
  89
  90 ALenum DirectOutMode{AL_FALSE};
  91 bool EnableWideStereo{false};
  92 bool EnableUhj{false};
  93 bool EnableSuperStereo{false};
  94 bool DisableVideo{false};
  95 LPALGETSOURCEI64VSOFT alGetSourcei64vSOFT;
  96 LPALCGETINTEGER64VSOFT alcGetInteger64vSOFT;
  97 LPALEVENTCONTROLSOFT alEventControlSOFT;
  98 LPALEVENTCALLBACKSOFT alEventCallbackSOFT;
  99
 100 LPALBUFFERCALLBACKSOFT alBufferCallbackSOFT;
 101
 102 const seconds AVNoSyncThreshold{10};
 103
 104 #define VIDEO_PICTURE_QUEUE_SIZE 24
 105
 106 const seconds_d64 AudioSyncThreshold{0.03};
 107 const milliseconds AudioSampleCorrectionMax{50};
 108 /* Averaging filter coefficient for audio sync. */
 109 #define AUDIO_DIFF_AVG_NB 20
 110 const double AudioAvgFilterCoeff{std::pow(0.01, 1.0/AUDIO_DIFF_AVG_NB)};
 111 /* Per-buffer size, in time */
 112 constexpr milliseconds AudioBufferTime{20};
 113 /* Buffer total size, in time (should be divisible by the buffer time) */
 114 constexpr milliseconds AudioBufferTotalTime{800};
 115 constexpr auto AudioBufferCount = AudioBufferTotalTime / AudioBufferTime;
 116
 117 enum {
 118     FF_MOVIE_DONE_EVENT = SDL_USEREVENT
 119 };
 120
 121 enum class SyncMaster {
 122     Audio,
 123     Video,
 124     External,
 125
 126     Default = Audio
 127 };
 128
 129
 130 inline microseconds get_avtime()
 131 { return microseconds{av_gettime()}; }
 132
 133 /* Define unique_ptrs to auto-cleanup associated ffmpeg objects. */
 134 struct AVIOContextDeleter {
 135     void operator()(AVIOContext *ptr) { avio_closep(&ptr); }
 136 };
 137 using AVIOContextPtr = std::unique_ptr<AVIOContext,AVIOContextDeleter>;
 138
 139 struct AVFormatCtxDeleter {
 140     void operator()(AVFormatContext *ptr) { avformat_close_input(&ptr); }
 141 };
 142 using AVFormatCtxPtr = std::unique_ptr<AVFormatContext,AVFormatCtxDeleter>;
 143
 144 struct AVCodecCtxDeleter {
 145     void operator()(AVCodecContext *ptr) { avcodec_free_context(&ptr); }
 146 };
 147 using AVCodecCtxPtr = std::unique_ptr<AVCodecContext,AVCodecCtxDeleter>;
 148
 149 struct AVPacketDeleter {
 150     void operator()(AVPacket *pkt) { av_packet_free(&pkt); }
 151 };
 152 using AVPacketPtr = std::unique_ptr<AVPacket,AVPacketDeleter>;
 153
 154 struct AVFrameDeleter {
 155     void operator()(AVFrame *ptr) { av_frame_free(&ptr); }
 156 };
 157 using AVFramePtr = std::unique_ptr<AVFrame,AVFrameDeleter>;
 158
 159 struct SwrContextDeleter {
 160     void operator()(SwrContext *ptr) { swr_free(&ptr); }
 161 };
 162 using SwrContextPtr = std::unique_ptr<SwrContext,SwrContextDeleter>;
 163
 164 struct SwsContextDeleter {
 165     void operator()(SwsContext *ptr) { sws_freeContext(ptr); }
 166 };
 167 using SwsContextPtr = std::unique_ptr<SwsContext,SwsContextDeleter>;
 168
 169
 170 template<size_t SizeLimit>
 171 class DataQueue {
 172     std::mutex mPacketMutex, mFrameMutex;
 173     std::condition_variable mPacketCond;
 174     std::condition_variable mInFrameCond, mOutFrameCond;
 175
 176     std::deque<AVPacketPtr> mPackets;
 177     size_t mTotalSize{0};
 178     bool mFinished{false};
 179
 180     AVPacketPtr getPacket()
 181     {
 182         std::unique_lock<std::mutex> plock{mPacketMutex};
 183         while(mPackets.empty() && !mFinished)
 184             mPacketCond.wait(plock);
 185         if(mPackets.empty())
 186             return nullptr;
 187
 188         auto ret = std::move(mPackets.front());
 189         mPackets.pop_front();
 190         mTotalSize -= static_cast<unsigned int>(ret->size);
 191         return ret;
 192     }
 193
 194 public:
 195     int sendPacket(AVCodecContext *codecctx)
 196     {
 197         AVPacketPtr packet{getPacket()};
 198
 199         int ret{};
 200         {
 201             std::unique_lock<std::mutex> flock{mFrameMutex};
 202             while((ret=avcodec_send_packet(codecctx, packet.get())) == AVERROR(EAGAIN))
 203                 mInFrameCond.wait_for(flock, milliseconds{50});
 204         }
 205         mOutFrameCond.notify_one();
 206
 207         if(!packet)
 208         {
 209             if(!ret) return AVErrorEOF;
 210             std::cerr<< "Failed to send flush packet: "<<ret <<std::endl;
 211             return ret;
 212         }
 213         if(ret < 0)
 214             std::cerr<< "Failed to send packet: "<<ret <<std::endl;
 215         return ret;
 216     }
 217
 218     int receiveFrame(AVCodecContext *codecctx, AVFrame *frame)
 219     {
 220         int ret{};
 221         {
 222             std::unique_lock<std::mutex> flock{mFrameMutex};
 223             while((ret=avcodec_receive_frame(codecctx, frame)) == AVERROR(EAGAIN))
 224                 mOutFrameCond.wait_for(flock, milliseconds{50});
 225         }
 226         mInFrameCond.notify_one();
 227         return ret;
 228     }
 229
 230     void setFinished()
 231     {
 232         {
 233             std::lock_guard<std::mutex> _{mPacketMutex};
 234             mFinished = true;
 235         }
 236         mPacketCond.notify_one();
 237     }
 238
 239     void flush()
 240     {
 241         {
 242             std::lock_guard<std::mutex> _{mPacketMutex};
 243             mFinished = true;
 244
 245             mPackets.clear();
 246             mTotalSize = 0;
 247         }
 248         mPacketCond.notify_one();
 249     }
 250
 251     bool put(const AVPacket *pkt)
 252     {
 253         {
 254             std::unique_lock<std::mutex> lock{mPacketMutex};
 255             if(mTotalSize >= SizeLimit || mFinished)
 256                 return false;
 257
 258             mPackets.push_back(AVPacketPtr{av_packet_alloc()});
 259             if(av_packet_ref(mPackets.back().get(), pkt) != 0)
 260             {
 261                 mPackets.pop_back();
 262                 return true;
 263             }
 264
 265             mTotalSize += static_cast<unsigned int>(mPackets.back()->size);
 266         }
 267         mPacketCond.notify_one();
 268         return true;
 269     }
 270 };
 271
 272
 273 struct MovieState;
 274
 275 struct AudioState {
 276     MovieState &mMovie;
 277
 278     AVStream *mStream{nullptr};
 279     AVCodecCtxPtr mCodecCtx;
 280
 281     DataQueue<2*1024*1024> mQueue;
 282
 283     /* Used for clock difference average computation */
 284     seconds_d64 mClockDiffAvg{0};
 285
 286     /* Time of the next sample to be buffered */
 287     nanoseconds mCurrentPts{0};
 288
 289     /* Device clock time that the stream started at. */
 290     nanoseconds mDeviceStartTime{nanoseconds::min()};
 291
 292     /* Decompressed sample frame, and swresample context for conversion */
 293     AVFramePtr    mDecodedFrame;
 294     SwrContextPtr mSwresCtx;
 295
 296     /* Conversion format, for what gets fed to OpenAL */
 297     uint64_t       mDstChanLayout{0};
 298     AVSampleFormat mDstSampleFmt{AV_SAMPLE_FMT_NONE};
 299
 300     /* Storage of converted samples */
 301     uint8_t *mSamples{nullptr};
 302     int mSamplesLen{0}; /* In samples */
 303     int mSamplesPos{0};
 304     int mSamplesMax{0};
 305
 306     std::unique_ptr<uint8_t[]> mBufferData;
 307     size_t mBufferDataSize{0};
 308     std::atomic<size_t> mReadPos{0};
 309     std::atomic<size_t> mWritePos{0};
 310
 311     /* OpenAL format */
 312     ALenum mFormat{AL_NONE};
 313     ALuint mFrameSize{0};
 314
 315     std::mutex mSrcMutex;
 316     std::condition_variable mSrcCond;
 317     std::atomic_flag mConnected;
 318     ALuint mSource{0};
 319     std::array<ALuint,AudioBufferCount> mBuffers{};
 320     ALuint mBufferIdx{0};
 321
 322     AudioState(MovieState &movie) : mMovie(movie)
 323     { mConnected.test_and_set(std::memory_order_relaxed); }
 324     ~AudioState()
 325     {
 326         if(mSource)
 327             alDeleteSources(1, &mSource);
 328         if(mBuffers[0])
 329             alDeleteBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
 330
 331         av_freep(&mSamples);
 332     }
 333
 334     static void AL_APIENTRY eventCallbackC(ALenum eventType, ALuint object, ALuint param,
 335         ALsizei length, const ALchar *message, void *userParam)
 336     { static_cast<AudioState*>(userParam)->eventCallback(eventType, object, param, length, message); }
 337     void eventCallback(ALenum eventType, ALuint object, ALuint param, ALsizei length,
 338         const ALchar *message);
 339
 340     static ALsizei AL_APIENTRY bufferCallbackC(void *userptr, void *data, ALsizei size)
 341     { return static_cast<AudioState*>(userptr)->bufferCallback(data, size); }
 342     ALsizei bufferCallback(void *data, ALsizei size);
 343
 344     nanoseconds getClockNoLock();
 345     nanoseconds getClock()
 346     {
 347         std::lock_guard<std::mutex> lock{mSrcMutex};
 348         return getClockNoLock();
 349     }
 350
 351     bool startPlayback();
 352
 353     int getSync();
 354     int decodeFrame();
 355     bool readAudio(uint8_t *samples, unsigned int length, int &sample_skip);
 356     bool readAudio(int sample_skip);
 357
 358     int handler();
 359 };
 360
 361 struct VideoState {
 362     MovieState &mMovie;
 363
 364     AVStream *mStream{nullptr};
 365     AVCodecCtxPtr mCodecCtx;
 366
 367     DataQueue<14*1024*1024> mQueue;
 368
 369     /* The pts of the currently displayed frame, and the time (av_gettime) it
 370      * was last updated - used to have running video pts
 371      */
 372     nanoseconds mDisplayPts{0};
 373     microseconds mDisplayPtsTime{microseconds::min()};
 374     std::mutex mDispPtsMutex;
 375
 376     /* Swscale context for format conversion */
 377     SwsContextPtr mSwscaleCtx;
 378
 379     struct Picture {
 380         AVFramePtr mFrame{};
 381         nanoseconds mPts{nanoseconds::min()};
 382     };
 383     std::array<Picture,VIDEO_PICTURE_QUEUE_SIZE> mPictQ;
 384     std::atomic<size_t> mPictQRead{0u}, mPictQWrite{1u};
 385     std::mutex mPictQMutex;
 386     std::condition_variable mPictQCond;
 387
 388     SDL_Texture *mImage{nullptr};
 389     int mWidth{0}, mHeight{0}; /* Full texture size */
 390     bool mFirstUpdate{true};
 391
 392     std::atomic<bool> mEOS{false};
 393     std::atomic<bool> mFinalUpdate{false};
 394
 395     VideoState(MovieState &movie) : mMovie(movie) { }
 396     ~VideoState()
 397     {
 398         if(mImage)
 399             SDL_DestroyTexture(mImage);
 400         mImage = nullptr;
 401     }
 402
 403     nanoseconds getClock();
 404
 405     void display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame);
 406     void updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw);
 407     int handler();
 408 };
 409
 410 struct MovieState {
 411     AVIOContextPtr mIOContext;
 412     AVFormatCtxPtr mFormatCtx;
 413
 414     SyncMaster mAVSyncType{SyncMaster::Default};
 415
 416     microseconds mClockBase{microseconds::min()};
 417
 418     std::atomic<bool> mQuit{false};
 419
 420     AudioState mAudio;
 421     VideoState mVideo;
 422
 423     std::mutex mStartupMutex;
 424     std::condition_variable mStartupCond;
 425     bool mStartupDone{false};
 426
 427     std::thread mParseThread;
 428     std::thread mAudioThread;
 429     std::thread mVideoThread;
 430
 431     std::string mFilename;
 432
 433     MovieState(std::string fname)
 434       : mAudio(*this), mVideo(*this), mFilename(std::move(fname))
 435     { }
 436     ~MovieState()
 437     {
 438         stop();
 439         if(mParseThread.joinable())
 440             mParseThread.join();
 441     }
 442
 443     static int decode_interrupt_cb(void *ctx);
 444     bool prepare();
 445     void setTitle(SDL_Window *window);
 446     void stop();
 447
 448     nanoseconds getClock();
 449
 450     nanoseconds getMasterClock();
 451
 452     nanoseconds getDuration();
 453
 454     int streamComponentOpen(unsigned int stream_index);
 455     int parse_handler();
 456 };
 457
 458
 459 nanoseconds AudioState::getClockNoLock()
 460 {
 461     // The audio clock is the timestamp of the sample currently being heard.
 462     if(alcGetInteger64vSOFT)
 463     {
 464         // If device start time = min, we aren't playing yet.
 465         if(mDeviceStartTime == nanoseconds::min())
 466             return nanoseconds::zero();
 467
 468         // Get the current device clock time and latency.
 469         auto device = alcGetContextsDevice(alcGetCurrentContext());
 470         ALCint64SOFT devtimes[2]{0,0};
 471         alcGetInteger64vSOFT(device, ALC_DEVICE_CLOCK_LATENCY_SOFT, 2, devtimes);
 472         auto latency = nanoseconds{devtimes[1]};
 473         auto device_time = nanoseconds{devtimes[0]};
 474
 475         // The clock is simply the current device time relative to the recorded
 476         // start time. We can also subtract the latency to get more a accurate
 477         // position of where the audio device actually is in the output stream.
 478         return device_time - mDeviceStartTime - latency;
 479     }
 480
 481     if(mBufferDataSize > 0)
 482     {
 483         if(mDeviceStartTime == nanoseconds::min())
 484             return nanoseconds::zero();
 485
 486         /* With a callback buffer and no device clock, mDeviceStartTime is
 487          * actually the timestamp of the first sample frame played. The audio
 488          * clock, then, is that plus the current source offset.
 489          */
 490         ALint64SOFT offset[2];
 491         if(alGetSourcei64vSOFT)
 492             alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
 493         else
 494         {
 495             ALint ioffset;
 496             alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
 497             offset[0] = ALint64SOFT{ioffset} << 32;
 498             offset[1] = 0;
 499         }
 500         /* NOTE: The source state must be checked last, in case an underrun
 501          * occurs and the source stops between getting the state and retrieving
 502          * the offset+latency.
 503          */
 504         ALint status;
 505         alGetSourcei(mSource, AL_SOURCE_STATE, &status);
 506
 507         nanoseconds pts{};
 508         if(status == AL_PLAYING || status == AL_PAUSED)
 509             pts = mDeviceStartTime - nanoseconds{offset[1]} +
 510                 duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
 511         else
 512         {
 513             /* If the source is stopped, the pts of the next sample to be heard
 514              * is the pts of the next sample to be buffered, minus the amount
 515              * already in the buffer ready to play.
 516              */
 517             const size_t woffset{mWritePos.load(std::memory_order_acquire)};
 518             const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
 519             const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
 520                 roffset};
 521
 522             pts = mCurrentPts - nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
 523         }
 524
 525         return pts;
 526     }
 527
 528     /* The source-based clock is based on 4 components:
 529      * 1 - The timestamp of the next sample to buffer (mCurrentPts)
 530      * 2 - The length of the source's buffer queue
 531      *     (AudioBufferTime*AL_BUFFERS_QUEUED)
 532      * 3 - The offset OpenAL is currently at in the source (the first value
 533      *     from AL_SAMPLE_OFFSET_LATENCY_SOFT)
 534      * 4 - The latency between OpenAL and the DAC (the second value from
 535      *     AL_SAMPLE_OFFSET_LATENCY_SOFT)
 536      *
 537      * Subtracting the length of the source queue from the next sample's
 538      * timestamp gives the timestamp of the sample at the start of the source
 539      * queue. Adding the source offset to that results in the timestamp for the
 540      * sample at OpenAL's current position, and subtracting the source latency
 541      * from that gives the timestamp of the sample currently at the DAC.
 542      */
 543     nanoseconds pts{mCurrentPts};
 544     if(mSource)
 545     {
 546         ALint64SOFT offset[2];
 547         if(alGetSourcei64vSOFT)
 548             alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
 549         else
 550         {
 551             ALint ioffset;
 552             alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
 553             offset[0] = ALint64SOFT{ioffset} << 32;
 554             offset[1] = 0;
 555         }
 556         ALint queued, status;
 557         alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
 558         alGetSourcei(mSource, AL_SOURCE_STATE, &status);
 559
 560         /* If the source is AL_STOPPED, then there was an underrun and all
 561          * buffers are processed, so ignore the source queue. The audio thread
 562          * will put the source into an AL_INITIAL state and clear the queue
 563          * when it starts recovery.
 564          */
 565         if(status != AL_STOPPED)
 566         {
 567             pts -= AudioBufferTime*queued;
 568             pts += duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
 569         }
 570         /* Don't offset by the latency if the source isn't playing. */
 571         if(status == AL_PLAYING)
 572             pts -= nanoseconds{offset[1]};
 573     }
 574
 575     return std::max(pts, nanoseconds::zero());
 576 }
 577
 578 bool AudioState::startPlayback()
 579 {
 580     const size_t woffset{mWritePos.load(std::memory_order_acquire)};
 581     const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
 582     const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
 583         roffset};
 584
 585     if(mBufferDataSize > 0)
 586     {
 587         if(readable == 0)
 588             return false;
 589         if(!alcGetInteger64vSOFT)
 590             mDeviceStartTime = mCurrentPts -
 591                 nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
 592     }
 593     else
 594     {
 595         ALint queued{};
 596         alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
 597         if(queued == 0) return false;
 598     }
 599
 600     alSourcePlay(mSource);
 601     if(alcGetInteger64vSOFT)
 602     {
 603         /* Subtract the total buffer queue time from the current pts to get the
 604          * pts of the start of the queue.
 605          */
 606         int64_t srctimes[2]{0,0};
 607         alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_CLOCK_SOFT, srctimes);
 608         auto device_time = nanoseconds{srctimes[1]};
 609         auto src_offset = duration_cast<nanoseconds>(fixed32{srctimes[0]}) /
 610             mCodecCtx->sample_rate;
 611
 612         /* The mixer may have ticked and incremented the device time and sample
 613          * offset, so subtract the source offset from the device time to get
 614          * the device time the source started at. Also subtract startpts to get
 615          * the device time the stream would have started at to reach where it
 616          * is now.
 617          */
 618         if(mBufferDataSize > 0)
 619         {
 620             nanoseconds startpts{mCurrentPts -
 621                 nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate};
 622             mDeviceStartTime = device_time - src_offset - startpts;
 623         }
 624         else
 625         {
 626             nanoseconds startpts{mCurrentPts - AudioBufferTotalTime};
 627             mDeviceStartTime = device_time - src_offset - startpts;
 628         }
 629     }
 630     return true;
 631 }
 632
 633 int AudioState::getSync()
 634 {
 635     if(mMovie.mAVSyncType == SyncMaster::Audio)
 636         return 0;
 637
 638     auto ref_clock = mMovie.getMasterClock();
 639     auto diff = ref_clock - getClockNoLock();
 640
 641     if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
 642     {
 643         /* Difference is TOO big; reset accumulated average */
 644         mClockDiffAvg = seconds_d64::zero();
 645         return 0;
 646     }
 647
 648     /* Accumulate the diffs */
 649     mClockDiffAvg = mClockDiffAvg*AudioAvgFilterCoeff + diff;
 650     auto avg_diff = mClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
 651     if(avg_diff < AudioSyncThreshold/2.0 && avg_diff > -AudioSyncThreshold)
 652         return 0;
 653
 654     /* Constrain the per-update difference to avoid exceedingly large skips */
 655     diff = std::min<nanoseconds>(diff, AudioSampleCorrectionMax);
 656     return static_cast<int>(duration_cast<seconds>(diff*mCodecCtx->sample_rate).count());
 657 }
 658
 659 int AudioState::decodeFrame()
 660 {
 661     do {
 662         while(int ret{mQueue.receiveFrame(mCodecCtx.get(), mDecodedFrame.get())})
 663         {
 664             if(ret == AVErrorEOF) return 0;
 665             std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
 666         }
 667     } while(mDecodedFrame->nb_samples <= 0);
 668
 669     /* If provided, update w/ pts */
 670     if(mDecodedFrame->best_effort_timestamp != AVNoPtsValue)
 671         mCurrentPts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
 672             static_cast<double>(mDecodedFrame->best_effort_timestamp)});
 673
 674     if(mDecodedFrame->nb_samples > mSamplesMax)
 675     {
 676         av_freep(&mSamples);
 677         av_samples_alloc(&mSamples, nullptr, mCodecCtx->channels, mDecodedFrame->nb_samples,
 678             mDstSampleFmt, 0);
 679         mSamplesMax = mDecodedFrame->nb_samples;
 680     }
 681     /* Return the amount of sample frames converted */
 682     int data_size{swr_convert(mSwresCtx.get(), &mSamples, mDecodedFrame->nb_samples,
 683         const_cast<const uint8_t**>(mDecodedFrame->data), mDecodedFrame->nb_samples)};
 684
 685     av_frame_unref(mDecodedFrame.get());
 686     return data_size;
 687 }
 688
 689 /* Duplicates the sample at in to out, count times. The frame size is a
 690  * multiple of the template type size.
 691  */
 692 template<typename T>
 693 static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
 694 {
 695     auto *sample = reinterpret_cast<const T*>(in);
 696     auto *dst = reinterpret_cast<T*>(out);
 697
 698     /* NOTE: frame_size is a multiple of sizeof(T). */
 699     size_t type_mult{frame_size / sizeof(T)};
 700     if(type_mult == 1)
 701         std::fill_n(dst, count, *sample);
 702     else for(size_t i{0};i < count;++i)
 703     {
 704         for(size_t j{0};j < type_mult;++j)
 705             dst[i*type_mult + j] = sample[j];
 706     }
 707 }
 708
 709 static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
 710 {
 711     if((frame_size&7) == 0)
 712         sample_dup<uint64_t>(out, in, count, frame_size);
 713     else if((frame_size&3) == 0)
 714         sample_dup<uint32_t>(out, in, count, frame_size);
 715     else if((frame_size&1) == 0)
 716         sample_dup<uint16_t>(out, in, count, frame_size);
 717     else
 718         sample_dup<uint8_t>(out, in, count, frame_size);
 719 }
 720
 721 bool AudioState::readAudio(uint8_t *samples, unsigned int length, int &sample_skip)
 722 {
 723     unsigned int audio_size{0};
 724
 725     /* Read the next chunk of data, refill the buffer, and queue it
 726      * on the source */
 727     length /= mFrameSize;
 728     while(mSamplesLen > 0 && audio_size < length)
 729     {
 730         unsigned int rem{length - audio_size};
 731         if(mSamplesPos >= 0)
 732         {
 733             const auto len = static_cast<unsigned int>(mSamplesLen - mSamplesPos);
 734             if(rem > len) rem = len;
 735             std::copy_n(mSamples + static_cast<unsigned int>(mSamplesPos)*mFrameSize,
 736                 rem*mFrameSize, samples);
 737         }
 738         else
 739         {
 740             rem = std::min(rem, static_cast<unsigned int>(-mSamplesPos));
 741
 742             /* Add samples by copying the first sample */
 743             sample_dup(samples, mSamples, rem, mFrameSize);
 744         }
 745
 746         mSamplesPos += rem;
 747         mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 748         samples += rem*mFrameSize;
 749         audio_size += rem;
 750
 751         while(mSamplesPos >= mSamplesLen)
 752         {
 753             mSamplesLen = decodeFrame();
 754             mSamplesPos = std::min(mSamplesLen, sample_skip);
 755             if(mSamplesLen <= 0) break;
 756
 757             sample_skip -= mSamplesPos;
 758
 759             // Adjust the device start time and current pts by the amount we're
 760             // skipping/duplicating, so that the clock remains correct for the
 761             // current stream position.
 762             auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
 763             mDeviceStartTime -= skip;
 764             mCurrentPts += skip;
 765         }
 766     }
 767     if(audio_size <= 0)
 768         return false;
 769
 770     if(audio_size < length)
 771     {
 772         const unsigned int rem{length - audio_size};
 773         std::fill_n(samples, rem*mFrameSize,
 774             (mDstSampleFmt == AV_SAMPLE_FMT_U8) ? 0x80 : 0x00);
 775         mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 776     }
 777     return true;
 778 }
 779
 780 bool AudioState::readAudio(int sample_skip)
 781 {
 782     size_t woffset{mWritePos.load(std::memory_order_acquire)};
 783     const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
 784     while(mSamplesLen > 0)
 785     {
 786         const size_t nsamples{((roffset > woffset) ? roffset-woffset-1
 787             : (roffset == 0) ? (mBufferDataSize-woffset-1)
 788             : (mBufferDataSize-woffset)) / mFrameSize};
 789         if(!nsamples) break;
 790
 791         if(mSamplesPos < 0)
 792         {
 793             const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(-mSamplesPos))};
 794
 795             sample_dup(&mBufferData[woffset], mSamples, rem, mFrameSize);
 796             woffset += rem * mFrameSize;
 797             if(woffset == mBufferDataSize) woffset = 0;
 798             mWritePos.store(woffset, std::memory_order_release);
 799
 800             mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 801             mSamplesPos += static_cast<int>(rem);
 802             continue;
 803         }
 804
 805         const size_t rem{std::min<size_t>(nsamples, static_cast<ALuint>(mSamplesLen-mSamplesPos))};
 806         const size_t boffset{static_cast<ALuint>(mSamplesPos) * size_t{mFrameSize}};
 807         const size_t nbytes{rem * mFrameSize};
 808
 809         memcpy(&mBufferData[woffset], mSamples + boffset, nbytes);
 810         woffset += nbytes;
 811         if(woffset == mBufferDataSize) woffset = 0;
 812         mWritePos.store(woffset, std::memory_order_release);
 813
 814         mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 815         mSamplesPos += static_cast<int>(rem);
 816
 817         while(mSamplesPos >= mSamplesLen)
 818         {
 819             mSamplesLen = decodeFrame();
 820             mSamplesPos = std::min(mSamplesLen, sample_skip);
 821             if(mSamplesLen <= 0) return false;
 822
 823             sample_skip -= mSamplesPos;
 824
 825             auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
 826             mDeviceStartTime -= skip;
 827             mCurrentPts += skip;
 828         }
 829     }
 830
 831     return true;
 832 }
 833
 834
 835 void AL_APIENTRY AudioState::eventCallback(ALenum eventType, ALuint object, ALuint param,
 836     ALsizei length, const ALchar *message)
 837 {
 838     if(eventType == AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT)
 839     {
 840         /* Temporarily lock the source mutex to ensure it's not between
 841          * checking the processed count and going to sleep.
 842          */
 843         std::unique_lock<std::mutex>{mSrcMutex}.unlock();
 844         mSrcCond.notify_one();
 845         return;
 846     }
 847
 848     std::cout<< "\n---- AL Event on AudioState "<<this<<" ----\nEvent: ";
 849     switch(eventType)
 850     {
 851     case AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT: std::cout<< "Buffer completed"; break;
 852     case AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT: std::cout<< "Source state changed"; break;
 853     case AL_EVENT_TYPE_DISCONNECTED_SOFT: std::cout<< "Disconnected"; break;
 854     default:
 855         std::cout<< "0x"<<std::hex<<std::setw(4)<<std::setfill('0')<<eventType<<std::dec<<
 856             std::setw(0)<<std::setfill(' '); break;
 857     }
 858     std::cout<< "\n"
 859         "Object ID: "<<object<<"\n"
 860         "Parameter: "<<param<<"\n"
 861         "Message: "<<std::string{message, static_cast<ALuint>(length)}<<"\n----"<<
 862         std::endl;
 863
 864     if(eventType == AL_EVENT_TYPE_DISCONNECTED_SOFT)
 865     {
 866         {
 867             std::lock_guard<std::mutex> lock{mSrcMutex};
 868             mConnected.clear(std::memory_order_release);
 869         }
 870         mSrcCond.notify_one();
 871     }
 872 }
 873
 874 ALsizei AudioState::bufferCallback(void *data, ALsizei size)
 875 {
 876     ALsizei got{0};
 877
 878     size_t roffset{mReadPos.load(std::memory_order_acquire)};
 879     while(got < size)
 880     {
 881         const size_t woffset{mWritePos.load(std::memory_order_relaxed)};
 882         if(woffset == roffset) break;
 883
 884         size_t todo{((woffset < roffset) ? mBufferDataSize : woffset) - roffset};
 885         todo = std::min<size_t>(todo, static_cast<ALuint>(size-got));
 886
 887         memcpy(data, &mBufferData[roffset], todo);
 888         data = static_cast<ALbyte*>(data) + todo;
 889         got += static_cast<ALsizei>(todo);
 890
 891         roffset += todo;
 892         if(roffset == mBufferDataSize)
 893             roffset = 0;
 894     }
 895     mReadPos.store(roffset, std::memory_order_release);
 896
 897     return got;
 898 }
 899
 900 int AudioState::handler()
 901 {
 902     std::unique_lock<std::mutex> srclock{mSrcMutex, std::defer_lock};
 903     milliseconds sleep_time{AudioBufferTime / 3};
 904
 905     struct EventControlManager {
 906         const std::array<ALenum,3> evt_types{{
 907             AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT, AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT,
 908             AL_EVENT_TYPE_DISCONNECTED_SOFT}};
 909
 910         EventControlManager(milliseconds &sleep_time)
 911         {
 912             if(alEventControlSOFT)
 913             {
 914                 alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
 915                     AL_TRUE);
 916                 alEventCallbackSOFT(&AudioState::eventCallbackC, this);
 917                 sleep_time = AudioBufferTotalTime;
 918             }
 919         }
 920         ~EventControlManager()
 921         {
 922             if(alEventControlSOFT)
 923             {
 924                 alEventControlSOFT(static_cast<ALsizei>(evt_types.size()), evt_types.data(),
 925                     AL_FALSE);
 926                 alEventCallbackSOFT(nullptr, nullptr);
 927             }
 928         }
 929     };
 930     EventControlManager event_controller{sleep_time};
 931
 932     const bool has_bfmt_ex{alIsExtensionPresent("AL_SOFT_bformat_ex") != AL_FALSE};
 933     ALenum ambi_layout{AL_FUMA_SOFT};
 934     ALenum ambi_scale{AL_FUMA_SOFT};
 935
 936     std::unique_ptr<uint8_t[]> samples;
 937     ALsizei buffer_len{0};
 938
 939     /* Find a suitable format for OpenAL. */
 940     mDstChanLayout = 0;
 941     mFormat = AL_NONE;
 942     if((mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLT || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLTP
 943             || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBL
 944             || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_DBLP
 945             || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32
 946             || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S32P
 947             || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64
 948             || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_S64P)
 949         && alIsExtensionPresent("AL_EXT_FLOAT32"))
 950     {
 951         mDstSampleFmt = AV_SAMPLE_FMT_FLT;
 952         mFrameSize = 4;
 953         if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
 954         {
 955             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1)
 956             {
 957                 mDstChanLayout = mCodecCtx->channel_layout;
 958                 mFrameSize *= 8;
 959                 mFormat = alGetEnumValue("AL_FORMAT_71CHN32");
 960             }
 961             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1
 962                 || mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK)
 963             {
 964                 mDstChanLayout = mCodecCtx->channel_layout;
 965                 mFrameSize *= 6;
 966                 mFormat = alGetEnumValue("AL_FORMAT_51CHN32");
 967             }
 968             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_QUAD)
 969             {
 970                 mDstChanLayout = mCodecCtx->channel_layout;
 971                 mFrameSize *= 4;
 972                 mFormat = alGetEnumValue("AL_FORMAT_QUAD32");
 973             }
 974         }
 975         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
 976         {
 977             mDstChanLayout = mCodecCtx->channel_layout;
 978             mFrameSize *= 1;
 979             mFormat = AL_FORMAT_MONO_FLOAT32;
 980         }
 981         /* Assume 3D B-Format (ambisonics) if the channel layout is blank and
 982          * there's 4 or more channels. FFmpeg/libavcodec otherwise seems to
 983          * have no way to specify if the source is actually B-Format (let alone
 984          * if it's 2D or 3D).
 985          */
 986         if(mCodecCtx->channel_layout == 0 && mCodecCtx->channels >= 4
 987             && alIsExtensionPresent("AL_EXT_BFORMAT"))
 988         {
 989             /* Calculate what should be the ambisonic order from the number of
 990              * channels, and confirm that's the number of channels. Opus allows
 991              * an optional non-diegetic stereo stream with the B-Format stream,
 992              * which we can ignore, so check for that too.
 993              */
 994             auto order = static_cast<int>(std::sqrt(mCodecCtx->channels)) - 1;
 995             int channels{(order+1) * (order+1)};
 996             if(channels == mCodecCtx->channels || channels+2 == mCodecCtx->channels)
 997             {
 998                 /* OpenAL only supports first-order with AL_EXT_BFORMAT, which
 999                  * is 4 channels for 3D buffers.
1000                  */
1001                 mFrameSize *= 4;
1002                 mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_FLOAT32");
1003             }
1004         }
1005         if(!mFormat || mFormat == -1)
1006         {
1007             mDstChanLayout = AV_CH_LAYOUT_STEREO;
1008             mFrameSize *= 2;
1009             mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN_FLOAT32_SOFT : AL_FORMAT_STEREO_FLOAT32;
1010         }
1011     }
1012     if(mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8P)
1013     {
1014         mDstSampleFmt = AV_SAMPLE_FMT_U8;
1015         mFrameSize = 1;
1016         if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
1017         {
1018             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1)
1019             {
1020                 mDstChanLayout = mCodecCtx->channel_layout;
1021                 mFrameSize *= 8;
1022                 mFormat = alGetEnumValue("AL_FORMAT_71CHN8");
1023             }
1024             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1
1025                 || mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK)
1026             {
1027                 mDstChanLayout = mCodecCtx->channel_layout;
1028                 mFrameSize *= 6;
1029                 mFormat = alGetEnumValue("AL_FORMAT_51CHN8");
1030             }
1031             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_QUAD)
1032             {
1033                 mDstChanLayout = mCodecCtx->channel_layout;
1034                 mFrameSize *= 4;
1035                 mFormat = alGetEnumValue("AL_FORMAT_QUAD8");
1036             }
1037         }
1038         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
1039         {
1040             mDstChanLayout = mCodecCtx->channel_layout;
1041             mFrameSize *= 1;
1042             mFormat = AL_FORMAT_MONO8;
1043         }
1044         if(mCodecCtx->channel_layout == 0 && mCodecCtx->channels >= 4
1045             && alIsExtensionPresent("AL_EXT_BFORMAT"))
1046         {
1047             auto order = static_cast<int>(std::sqrt(mCodecCtx->channels)) - 1;
1048             int channels{(order+1) * (order+1)};
1049             if(channels == mCodecCtx->channels || channels+2 == mCodecCtx->channels)
1050             {
1051                 mFrameSize *= 4;
1052                 mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_8");
1053             }
1054         }
1055         if(!mFormat || mFormat == -1)
1056         {
1057             mDstChanLayout = AV_CH_LAYOUT_STEREO;
1058             mFrameSize *= 2;
1059             mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN8_SOFT : AL_FORMAT_STEREO8;
1060         }
1061     }
1062     if(!mFormat || mFormat == -1)
1063     {
1064         mDstSampleFmt = AV_SAMPLE_FMT_S16;
1065         mFrameSize = 2;
1066         if(alIsExtensionPresent("AL_EXT_MCFORMATS"))
1067         {
1068             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1)
1069             {
1070                 mDstChanLayout = mCodecCtx->channel_layout;
1071                 mFrameSize *= 8;
1072                 mFormat = alGetEnumValue("AL_FORMAT_71CHN16");
1073             }
1074             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1
1075                 || mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK)
1076             {
1077                 mDstChanLayout = mCodecCtx->channel_layout;
1078                 mFrameSize *= 6;
1079                 mFormat = alGetEnumValue("AL_FORMAT_51CHN16");
1080             }
1081             if(mCodecCtx->channel_layout == AV_CH_LAYOUT_QUAD)
1082             {
1083                 mDstChanLayout = mCodecCtx->channel_layout;
1084                 mFrameSize *= 4;
1085                 mFormat = alGetEnumValue("AL_FORMAT_QUAD16");
1086             }
1087         }
1088         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
1089         {
1090             mDstChanLayout = mCodecCtx->channel_layout;
1091             mFrameSize *= 1;
1092             mFormat = AL_FORMAT_MONO16;
1093         }
1094         if(mCodecCtx->channel_layout == 0 && mCodecCtx->channels >= 4
1095             && alIsExtensionPresent("AL_EXT_BFORMAT"))
1096         {
1097             auto order = static_cast<int>(std::sqrt(mCodecCtx->channels)) - 1;
1098             int channels{(order+1) * (order+1)};
1099             if(channels == mCodecCtx->channels || channels+2 == mCodecCtx->channels)
1100             {
1101                 mFrameSize *= 4;
1102                 mFormat = alGetEnumValue("AL_FORMAT_BFORMAT3D_16");
1103             }
1104         }
1105         if(!mFormat || mFormat == -1)
1106         {
1107             mDstChanLayout = AV_CH_LAYOUT_STEREO;
1108             mFrameSize *= 2;
1109             mFormat = EnableUhj ? AL_FORMAT_UHJ2CHN16_SOFT : AL_FORMAT_STEREO16;
1110         }
1111     }
1112
1113     mSamples = nullptr;
1114     mSamplesMax = 0;
1115     mSamplesPos = 0;
1116     mSamplesLen = 0;
1117
1118     mDecodedFrame.reset(av_frame_alloc());
1119     if(!mDecodedFrame)
1120     {
1121         std::cerr<< "Failed to allocate audio frame" <<std::endl;
1122         return 0;
1123     }
1124
1125     if(!mDstChanLayout)
1126     {
1127         /* OpenAL only supports first-order ambisonics with AL_EXT_BFORMAT, so
1128          * we have to drop any extra channels.
1129          */
1130         mSwresCtx.reset(swr_alloc_set_opts(nullptr,
1131             (1_i64<<4)-1, mDstSampleFmt, mCodecCtx->sample_rate,
1132             (1_i64<<mCodecCtx->channels)-1, mCodecCtx->sample_fmt, mCodecCtx->sample_rate,
1133             0, nullptr));
1134
1135         /* Note that ffmpeg/libavcodec has no method to check the ambisonic
1136          * channel order and normalization, so we can only assume AmbiX as the
1137          * defacto-standard. This is not true for .amb files, which use FuMa.
1138          */
1139         std::vector<double> mtx(64*64, 0.0);
1140         ambi_layout = AL_ACN_SOFT;
1141         ambi_scale = AL_SN3D_SOFT;
1142         if(has_bfmt_ex)
1143         {
1144             /* An identity matrix that doesn't remix any channels. */
1145             std::cout<< "Found AL_SOFT_bformat_ex" <<std::endl;
1146             mtx[0 + 0*64] = 1.0;
1147             mtx[1 + 1*64] = 1.0;
1148             mtx[2 + 2*64] = 1.0;
1149             mtx[3 + 3*64] = 1.0;
1150         }
1151         else
1152         {
1153             std::cout<< "Found AL_EXT_BFORMAT" <<std::endl;
1154             /* Without AL_SOFT_bformat_ex, OpenAL only supports FuMa channel
1155              * ordering and normalization, so a custom matrix is needed to
1156              * scale and reorder the source from AmbiX.
1157              */
1158             mtx[0 + 0*64] = std::sqrt(0.5);
1159             mtx[3 + 1*64] = 1.0;
1160             mtx[1 + 2*64] = 1.0;
1161             mtx[2 + 3*64] = 1.0;
1162         }
1163         swr_set_matrix(mSwresCtx.get(), mtx.data(), 64);
1164     }
1165     else
1166         mSwresCtx.reset(swr_alloc_set_opts(nullptr,
1167             static_cast<int64_t>(mDstChanLayout), mDstSampleFmt, mCodecCtx->sample_rate,
1168             mCodecCtx->channel_layout ? static_cast<int64_t>(mCodecCtx->channel_layout)
1169                 : av_get_default_channel_layout(mCodecCtx->channels),
1170             mCodecCtx->sample_fmt, mCodecCtx->sample_rate,
1171             0, nullptr));
1172     if(!mSwresCtx || swr_init(mSwresCtx.get()) != 0)
1173     {
1174         std::cerr<< "Failed to initialize audio converter" <<std::endl;
1175         return 0;
1176     }
1177
1178     alGenBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
1179     alGenSources(1, &mSource);
1180
1181     if(DirectOutMode)
1182         alSourcei(mSource, AL_DIRECT_CHANNELS_SOFT, DirectOutMode);
1183     if(EnableWideStereo)
1184     {
1185         const float angles[2]{static_cast<float>(M_PI / 3.0), static_cast<float>(-M_PI / 3.0)};
1186         alSourcefv(mSource, AL_STEREO_ANGLES, angles);
1187     }
1188     if(has_bfmt_ex)
1189     {
1190         for(ALuint bufid : mBuffers)
1191         {
1192             alBufferi(bufid, AL_AMBISONIC_LAYOUT_SOFT, ambi_layout);
1193             alBufferi(bufid, AL_AMBISONIC_SCALING_SOFT, ambi_scale);
1194         }
1195     }
1196 #ifdef AL_SOFT_UHJ
1197     if(EnableSuperStereo)
1198         alSourcei(mSource, AL_STEREO_MODE_SOFT, AL_SUPER_STEREO_SOFT);
1199 #endif
1200
1201     if(alGetError() != AL_NO_ERROR)
1202         return 0;
1203
1204     bool callback_ok{false};
1205     if(alBufferCallbackSOFT)
1206     {
1207         alBufferCallbackSOFT(mBuffers[0], mFormat, mCodecCtx->sample_rate, bufferCallbackC, this);
1208         alSourcei(mSource, AL_BUFFER, static_cast<ALint>(mBuffers[0]));
1209         if(alGetError() != AL_NO_ERROR)
1210         {
1211             fprintf(stderr, "Failed to set buffer callback\n");
1212             alSourcei(mSource, AL_BUFFER, 0);
1213         }
1214         else
1215         {
1216             mBufferDataSize = static_cast<size_t>(duration_cast<seconds>(mCodecCtx->sample_rate *
1217                 AudioBufferTotalTime).count()) * mFrameSize;
1218             mBufferData = std::make_unique<uint8_t[]>(mBufferDataSize);
1219             std::fill_n(mBufferData.get(), mBufferDataSize, uint8_t{});
1220
1221             mReadPos.store(0, std::memory_order_relaxed);
1222             mWritePos.store(mBufferDataSize/mFrameSize/2*mFrameSize, std::memory_order_relaxed);
1223
1224             ALCint refresh{};
1225             alcGetIntegerv(alcGetContextsDevice(alcGetCurrentContext()), ALC_REFRESH, 1, &refresh);
1226             sleep_time = milliseconds{seconds{1}} / refresh;
1227             callback_ok = true;
1228         }
1229     }
1230     if(!callback_ok)
1231         buffer_len = static_cast<int>(duration_cast<seconds>(mCodecCtx->sample_rate *
1232             AudioBufferTime).count() * mFrameSize);
1233     if(buffer_len > 0)
1234         samples = std::make_unique<uint8_t[]>(static_cast<ALuint>(buffer_len));
1235
1236     /* Prefill the codec buffer. */
1237     auto packet_sender = [this]()
1238     {
1239         while(1)
1240         {
1241             const int ret{mQueue.sendPacket(mCodecCtx.get())};
1242             if(ret == AVErrorEOF) break;
1243         }
1244     };
1245     auto sender = std::async(std::launch::async, packet_sender);
1246
1247     srclock.lock();
1248     if(alcGetInteger64vSOFT)
1249     {
1250         int64_t devtime{};
1251         alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()), ALC_DEVICE_CLOCK_SOFT,
1252             1, &devtime);
1253         mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
1254     }
1255
1256     mSamplesLen = decodeFrame();
1257     if(mSamplesLen > 0)
1258     {
1259         mSamplesPos = std::min(mSamplesLen, getSync());
1260
1261         auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
1262         mDeviceStartTime -= skip;
1263         mCurrentPts += skip;
1264     }
1265
1266     while(1)
1267     {
1268         if(mMovie.mQuit.load(std::memory_order_relaxed))
1269         {
1270             /* If mQuit is set, drain frames until we can't get more audio,
1271              * indicating we've reached the flush packet and the packet sender
1272              * will also quit.
1273              */
1274             do {
1275                 mSamplesLen = decodeFrame();
1276                 mSamplesPos = mSamplesLen;
1277             } while(mSamplesLen > 0);
1278             goto finish;
1279         }
1280
1281         ALenum state;
1282         if(mBufferDataSize > 0)
1283         {
1284             alGetSourcei(mSource, AL_SOURCE_STATE, &state);
1285
1286             /* If mQuit is not set, don't quit even if there's no more audio,
1287              * so what's buffered has a chance to play to the real end.
1288              */
1289             readAudio(getSync());
1290         }
1291         else
1292         {
1293             ALint processed, queued;
1294
1295             /* First remove any processed buffers. */
1296             alGetSourcei(mSource, AL_BUFFERS_PROCESSED, &processed);
1297             while(processed > 0)
1298             {
1299                 ALuint bid;
1300                 alSourceUnqueueBuffers(mSource, 1, &bid);
1301                 --processed;
1302             }
1303
1304             /* Refill the buffer queue. */
1305             int sync_skip{getSync()};
1306             alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
1307             while(static_cast<ALuint>(queued) < mBuffers.size())
1308             {
1309                 /* Read the next chunk of data, filling the buffer, and queue
1310                  * it on the source.
1311                  */
1312                 if(!readAudio(samples.get(), static_cast<ALuint>(buffer_len), sync_skip))
1313                     break;
1314
1315                 const ALuint bufid{mBuffers[mBufferIdx]};
1316                 mBufferIdx = static_cast<ALuint>((mBufferIdx+1) % mBuffers.size());
1317
1318                 alBufferData(bufid, mFormat, samples.get(), buffer_len, mCodecCtx->sample_rate);
1319                 alSourceQueueBuffers(mSource, 1, &bufid);
1320                 ++queued;
1321             }
1322
1323             /* Check that the source is playing. */
1324             alGetSourcei(mSource, AL_SOURCE_STATE, &state);
1325             if(state == AL_STOPPED)
1326             {
1327                 /* AL_STOPPED means there was an underrun. Clear the buffer
1328                  * queue since this likely means we're late, and rewind the
1329                  * source to get it back into an AL_INITIAL state.
1330                  */
1331                 alSourceRewind(mSource);
1332                 alSourcei(mSource, AL_BUFFER, 0);
1333                 if(alcGetInteger64vSOFT)
1334                 {
1335                     /* Also update the device start time with the current
1336                      * device clock, so the decoder knows we're running behind.
1337                      */
1338                     int64_t devtime{};
1339                     alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()),
1340                         ALC_DEVICE_CLOCK_SOFT, 1, &devtime);
1341                     mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
1342                 }
1343                 continue;
1344             }
1345         }
1346
1347         /* (re)start the source if needed, and wait for a buffer to finish */
1348         if(state != AL_PLAYING && state != AL_PAUSED)
1349         {
1350             if(!startPlayback())
1351                 break;
1352         }
1353         if(ALenum err{alGetError()})
1354             std::cerr<< "Got AL error: 0x"<<std::hex<<err<<std::dec
1355                 << " ("<<alGetString(err)<<")" <<std::endl;
1356
1357         mSrcCond.wait_for(srclock, sleep_time);
1358     }
1359 finish:
1360
1361     alSourceRewind(mSource);
1362     alSourcei(mSource, AL_BUFFER, 0);
1363     srclock.unlock();
1364
1365     return 0;
1366 }
1367
1368
1369 nanoseconds VideoState::getClock()
1370 {
1371     /* NOTE: This returns incorrect times while not playing. */
1372     std::lock_guard<std::mutex> _{mDispPtsMutex};
1373     if(mDisplayPtsTime == microseconds::min())
1374         return nanoseconds::zero();
1375     auto delta = get_avtime() - mDisplayPtsTime;
1376     return mDisplayPts + delta;
1377 }
1378
1379 /* Called by VideoState::updateVideo to display the next video frame. */
1380 void VideoState::display(SDL_Window *screen, SDL_Renderer *renderer, AVFrame *frame)
1381 {
1382     if(!mImage)
1383         return;
1384
1385     double aspect_ratio;
1386     int win_w, win_h;
1387     int w, h, x, y;
1388
1389     int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
1390     int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
1391     if(frame->sample_aspect_ratio.num == 0)
1392         aspect_ratio = 0.0;
1393     else
1394     {
1395         aspect_ratio = av_q2d(frame->sample_aspect_ratio) * frame_width /
1396             frame_height;
1397     }
1398     if(aspect_ratio <= 0.0)
1399         aspect_ratio = static_cast<double>(frame_width) / frame_height;
1400
1401     SDL_GetWindowSize(screen, &win_w, &win_h);
1402     h = win_h;
1403     w = (static_cast<int>(std::rint(h * aspect_ratio)) + 3) & ~3;
1404     if(w > win_w)
1405     {
1406         w = win_w;
1407         h = (static_cast<int>(std::rint(w / aspect_ratio)) + 3) & ~3;
1408     }
1409     x = (win_w - w) / 2;
1410     y = (win_h - h) / 2;
1411
1412     SDL_Rect src_rect{ static_cast<int>(frame->crop_left), static_cast<int>(frame->crop_top),
1413         frame_width, frame_height };
1414     SDL_Rect dst_rect{ x, y, w, h };
1415     SDL_RenderCopy(renderer, mImage, &src_rect, &dst_rect);
1416     SDL_RenderPresent(renderer);
1417 }
1418
1419 /* Called regularly on the main thread where the SDL_Renderer was created. It
1420  * handles updating the textures of decoded frames and displaying the latest
1421  * frame.
1422  */
1423 void VideoState::updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw)
1424 {
1425     size_t read_idx{mPictQRead.load(std::memory_order_relaxed)};
1426     Picture *vp{&mPictQ[read_idx]};
1427
1428     auto clocktime = mMovie.getMasterClock();
1429     bool updated{false};
1430     while(1)
1431     {
1432         size_t next_idx{(read_idx+1)%mPictQ.size()};
1433         if(next_idx == mPictQWrite.load(std::memory_order_acquire))
1434             break;
1435         Picture *nextvp{&mPictQ[next_idx]};
1436         if(clocktime < nextvp->mPts && !mMovie.mQuit.load(std::memory_order_relaxed))
1437         {
1438             /* For the first update, ensure the first frame gets shown.  */
1439             if(!mFirstUpdate || updated)
1440                 break;
1441         }
1442
1443         vp = nextvp;
1444         updated = true;
1445         read_idx = next_idx;
1446     }
1447     if(mMovie.mQuit.load(std::memory_order_relaxed))
1448     {
1449         if(mEOS)
1450             mFinalUpdate = true;
1451         mPictQRead.store(read_idx, std::memory_order_release);
1452         std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1453         mPictQCond.notify_one();
1454         return;
1455     }
1456
1457     AVFrame *frame{vp->mFrame.get()};
1458     if(updated)
1459     {
1460         mPictQRead.store(read_idx, std::memory_order_release);
1461         std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1462         mPictQCond.notify_one();
1463
1464         /* allocate or resize the buffer! */
1465         bool fmt_updated{false};
1466         if(!mImage || mWidth != frame->width || mHeight != frame->height)
1467         {
1468             fmt_updated = true;
1469             if(mImage)
1470                 SDL_DestroyTexture(mImage);
1471             mImage = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING,
1472                 frame->width, frame->height);
1473             if(!mImage)
1474                 std::cerr<< "Failed to create YV12 texture!" <<std::endl;
1475             mWidth = frame->width;
1476             mHeight = frame->height;
1477         }
1478
1479         int frame_width{frame->width - static_cast<int>(frame->crop_left + frame->crop_right)};
1480         int frame_height{frame->height - static_cast<int>(frame->crop_top + frame->crop_bottom)};
1481         if(mFirstUpdate && frame_width > 0 && frame_height > 0)
1482         {
1483             /* For the first update, set the window size to the video size. */
1484             mFirstUpdate = false;
1485
1486             if(frame->sample_aspect_ratio.den != 0)
1487             {
1488                 double aspect_ratio = av_q2d(frame->sample_aspect_ratio);
1489                 if(aspect_ratio >= 1.0)
1490                     frame_width = static_cast<int>(frame_width*aspect_ratio + 0.5);
1491                 else if(aspect_ratio > 0.0)
1492                     frame_height = static_cast<int>(frame_height/aspect_ratio + 0.5);
1493             }
1494             SDL_SetWindowSize(screen, frame_width, frame_height);
1495         }
1496
1497         if(mImage)
1498         {
1499             void *pixels{nullptr};
1500             int pitch{0};
1501
1502             if(mCodecCtx->pix_fmt == AV_PIX_FMT_YUV420P)
1503                 SDL_UpdateYUVTexture(mImage, nullptr,
1504                     frame->data[0], frame->linesize[0],
1505                     frame->data[1], frame->linesize[1],
1506                     frame->data[2], frame->linesize[2]
1507                 );
1508             else if(SDL_LockTexture(mImage, nullptr, &pixels, &pitch) != 0)
1509                 std::cerr<< "Failed to lock texture" <<std::endl;
1510             else
1511             {
1512                 // Convert the image into YUV format that SDL uses
1513                 int w{frame->width};
1514                 int h{frame->height};
1515                 if(!mSwscaleCtx || fmt_updated)
1516                 {
1517                     mSwscaleCtx.reset(sws_getContext(
1518                         w, h, mCodecCtx->pix_fmt,
1519                         w, h, AV_PIX_FMT_YUV420P, 0,
1520                         nullptr, nullptr, nullptr
1521                     ));
1522                 }
1523
1524                 /* point pict at the queue */
1525                 uint8_t *pict_data[3];
1526                 pict_data[0] = static_cast<uint8_t*>(pixels);
1527                 pict_data[1] = pict_data[0] + w*h;
1528                 pict_data[2] = pict_data[1] + w*h/4;
1529
1530                 int pict_linesize[3];
1531                 pict_linesize[0] = pitch;
1532                 pict_linesize[1] = pitch / 2;
1533                 pict_linesize[2] = pitch / 2;
1534
1535                 sws_scale(mSwscaleCtx.get(), reinterpret_cast<uint8_t**>(frame->data), frame->linesize,
1536                     0, h, pict_data, pict_linesize);
1537                 SDL_UnlockTexture(mImage);
1538             }
1539
1540             redraw = true;
1541         }
1542     }
1543
1544     if(redraw)
1545     {
1546         /* Show the picture! */
1547         display(screen, renderer, frame);
1548     }
1549
1550     if(updated)
1551     {
1552         auto disp_time = get_avtime();
1553
1554         std::lock_guard<std::mutex> _{mDispPtsMutex};
1555         mDisplayPts = vp->mPts;
1556         mDisplayPtsTime = disp_time;
1557     }
1558     if(mEOS.load(std::memory_order_acquire))
1559     {
1560         if((read_idx+1)%mPictQ.size() == mPictQWrite.load(std::memory_order_acquire))
1561         {
1562             mFinalUpdate = true;
1563             std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1564             mPictQCond.notify_one();
1565         }
1566     }
1567 }
1568
1569 int VideoState::handler()
1570 {
1571     std::for_each(mPictQ.begin(), mPictQ.end(),
1572         [](Picture &pict) -> void
1573         { pict.mFrame = AVFramePtr{av_frame_alloc()}; });
1574
1575     /* Prefill the codec buffer. */
1576     auto packet_sender = [this]()
1577     {
1578         while(1)
1579         {
1580             const int ret{mQueue.sendPacket(mCodecCtx.get())};
1581             if(ret == AVErrorEOF) break;
1582         }
1583     };
1584     auto sender = std::async(std::launch::async, packet_sender);
1585
1586     {
1587         std::lock_guard<std::mutex> _{mDispPtsMutex};
1588         mDisplayPtsTime = get_avtime();
1589     }
1590
1591     auto current_pts = nanoseconds::zero();
1592     while(1)
1593     {
1594         size_t write_idx{mPictQWrite.load(std::memory_order_relaxed)};
1595         Picture *vp{&mPictQ[write_idx]};
1596
1597         /* Retrieve video frame. */
1598         AVFrame *decoded_frame{vp->mFrame.get()};
1599         while(int ret{mQueue.receiveFrame(mCodecCtx.get(), decoded_frame)})
1600         {
1601             if(ret == AVErrorEOF) goto finish;
1602             std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
1603         }
1604
1605         /* Get the PTS for this frame. */
1606         if(decoded_frame->best_effort_timestamp != AVNoPtsValue)
1607             current_pts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
1608                 static_cast<double>(decoded_frame->best_effort_timestamp)});
1609         vp->mPts = current_pts;
1610
1611         /* Update the video clock to the next expected PTS. */
1612         auto frame_delay = av_q2d(mCodecCtx->time_base);
1613         frame_delay += decoded_frame->repeat_pict * (frame_delay * 0.5);
1614         current_pts += duration_cast<nanoseconds>(seconds_d64{frame_delay});
1615
1616         /* Put the frame in the queue to be loaded into a texture and displayed
1617          * by the rendering thread.
1618          */
1619         write_idx = (write_idx+1)%mPictQ.size();
1620         mPictQWrite.store(write_idx, std::memory_order_release);
1621
1622         if(write_idx == mPictQRead.load(std::memory_order_acquire))
1623         {
1624             /* Wait until we have space for a new pic */
1625             std::unique_lock<std::mutex> lock{mPictQMutex};
1626             while(write_idx == mPictQRead.load(std::memory_order_acquire))
1627                 mPictQCond.wait(lock);
1628         }
1629     }
1630 finish:
1631     mEOS = true;
1632
1633     std::unique_lock<std::mutex> lock{mPictQMutex};
1634     while(!mFinalUpdate) mPictQCond.wait(lock);
1635
1636     return 0;
1637 }
1638
1639
1640 int MovieState::decode_interrupt_cb(void *ctx)
1641 {
1642     return static_cast<MovieState*>(ctx)->mQuit.load(std::memory_order_relaxed);
1643 }
1644
1645 bool MovieState::prepare()
1646 {
1647     AVIOContext *avioctx{nullptr};
1648     AVIOInterruptCB intcb{decode_interrupt_cb, this};
1649     if(avio_open2(&avioctx, mFilename.c_str(), AVIO_FLAG_READ, &intcb, nullptr))
1650     {
1651         std::cerr<< "Failed to open "<<mFilename <<std::endl;
1652         return false;
1653     }
1654     mIOContext.reset(avioctx);
1655
1656     /* Open movie file. If avformat_open_input fails it will automatically free
1657      * this context, so don't set it onto a smart pointer yet.
1658      */
1659     AVFormatContext *fmtctx{avformat_alloc_context()};
1660     fmtctx->pb = mIOContext.get();
1661     fmtctx->interrupt_callback = intcb;
1662     if(avformat_open_input(&fmtctx, mFilename.c_str(), nullptr, nullptr) != 0)
1663     {
1664         std::cerr<< "Failed to open "<<mFilename <<std::endl;
1665         return false;
1666     }
1667     mFormatCtx.reset(fmtctx);
1668
1669     /* Retrieve stream information */
1670     if(avformat_find_stream_info(mFormatCtx.get(), nullptr) < 0)
1671     {
1672         std::cerr<< mFilename<<": failed to find stream info" <<std::endl;
1673         return false;
1674     }
1675
1676     /* Dump information about file onto standard error */
1677     av_dump_format(mFormatCtx.get(), 0, mFilename.c_str(), 0);
1678
1679     mParseThread = std::thread{std::mem_fn(&MovieState::parse_handler), this};
1680
1681     std::unique_lock<std::mutex> slock{mStartupMutex};
1682     while(!mStartupDone) mStartupCond.wait(slock);
1683     return true;
1684 }
1685
1686 void MovieState::setTitle(SDL_Window *window)
1687 {
1688     auto pos1 = mFilename.rfind('/');
1689     auto pos2 = mFilename.rfind('\\');
1690     auto fpos = ((pos1 == std::string::npos) ? pos2 :
1691                  (pos2 == std::string::npos) ? pos1 :
1692                  std::max(pos1, pos2)) + 1;
1693     SDL_SetWindowTitle(window, (mFilename.substr(fpos)+" - "+AppName).c_str());
1694 }
1695
1696 nanoseconds MovieState::getClock()
1697 {
1698     if(mClockBase == microseconds::min())
1699         return nanoseconds::zero();
1700     return get_avtime() - mClockBase;
1701 }
1702
1703 nanoseconds MovieState::getMasterClock()
1704 {
1705     if(mAVSyncType == SyncMaster::Video && mVideo.mStream)
1706         return mVideo.getClock();
1707     if(mAVSyncType == SyncMaster::Audio && mAudio.mStream)
1708         return mAudio.getClock();
1709     return getClock();
1710 }
1711
1712 nanoseconds MovieState::getDuration()
1713 { return std::chrono::duration<int64_t,std::ratio<1,AV_TIME_BASE>>(mFormatCtx->duration); }
1714
1715 int MovieState::streamComponentOpen(unsigned int stream_index)
1716 {
1717     if(stream_index >= mFormatCtx->nb_streams)
1718         return -1;
1719
1720     /* Get a pointer to the codec context for the stream, and open the
1721      * associated codec.
1722      */
1723     AVCodecCtxPtr avctx{avcodec_alloc_context3(nullptr)};
1724     if(!avctx) return -1;
1725
1726     if(avcodec_parameters_to_context(avctx.get(), mFormatCtx->streams[stream_index]->codecpar))
1727         return -1;
1728
1729     const AVCodec *codec{avcodec_find_decoder(avctx->codec_id)};
1730     if(!codec || avcodec_open2(avctx.get(), codec, nullptr) < 0)
1731     {
1732         std::cerr<< "Unsupported codec: "<<avcodec_get_name(avctx->codec_id)
1733             << " (0x"<<std::hex<<avctx->codec_id<<std::dec<<")" <<std::endl;
1734         return -1;
1735     }
1736
1737     /* Initialize and start the media type handler */
1738     switch(avctx->codec_type)
1739     {
1740         case AVMEDIA_TYPE_AUDIO:
1741             mAudio.mStream = mFormatCtx->streams[stream_index];
1742             mAudio.mCodecCtx = std::move(avctx);
1743             break;
1744
1745         case AVMEDIA_TYPE_VIDEO:
1746             mVideo.mStream = mFormatCtx->streams[stream_index];
1747             mVideo.mCodecCtx = std::move(avctx);
1748             break;
1749
1750         default:
1751             return -1;
1752     }
1753
1754     return static_cast<int>(stream_index);
1755 }
1756
1757 int MovieState::parse_handler()
1758 {
1759     auto &audio_queue = mAudio.mQueue;
1760     auto &video_queue = mVideo.mQueue;
1761
1762     int video_index{-1};
1763     int audio_index{-1};
1764
1765     /* Find the first video and audio streams */
1766     for(unsigned int i{0u};i < mFormatCtx->nb_streams;i++)
1767     {
1768         auto codecpar = mFormatCtx->streams[i]->codecpar;
1769         if(codecpar->codec_type == AVMEDIA_TYPE_VIDEO && !DisableVideo && video_index < 0)
1770             video_index = streamComponentOpen(i);
1771         else if(codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_index < 0)
1772             audio_index = streamComponentOpen(i);
1773     }
1774
1775     {
1776         std::unique_lock<std::mutex> slock{mStartupMutex};
1777         mStartupDone = true;
1778     }
1779     mStartupCond.notify_all();
1780
1781     if(video_index < 0 && audio_index < 0)
1782     {
1783         std::cerr<< mFilename<<": could not open codecs" <<std::endl;
1784         mQuit = true;
1785     }
1786
1787     /* Set the base time 750ms ahead of the current av time. */
1788     mClockBase = get_avtime() + milliseconds{750};
1789
1790     if(audio_index >= 0)
1791         mAudioThread = std::thread{std::mem_fn(&AudioState::handler), &mAudio};
1792     if(video_index >= 0)
1793         mVideoThread = std::thread{std::mem_fn(&VideoState::handler), &mVideo};
1794
1795     /* Main packet reading/dispatching loop */
1796     AVPacketPtr packet{av_packet_alloc()};
1797     while(!mQuit.load(std::memory_order_relaxed))
1798     {
1799         if(av_read_frame(mFormatCtx.get(), packet.get()) < 0)
1800             break;
1801
1802         /* Copy the packet into the queue it's meant for. */
1803         if(packet->stream_index == video_index)
1804         {
1805             while(!mQuit.load(std::memory_order_acquire) && !video_queue.put(packet.get()))
1806                 std::this_thread::sleep_for(milliseconds{100});
1807         }
1808         else if(packet->stream_index == audio_index)
1809         {
1810             while(!mQuit.load(std::memory_order_acquire) && !audio_queue.put(packet.get()))
1811                 std::this_thread::sleep_for(milliseconds{100});
1812         }
1813
1814         av_packet_unref(packet.get());
1815     }
1816     /* Finish the queues so the receivers know nothing more is coming. */
1817     video_queue.setFinished();
1818     audio_queue.setFinished();
1819
1820     /* all done - wait for it */
1821     if(mVideoThread.joinable())
1822         mVideoThread.join();
1823     if(mAudioThread.joinable())
1824         mAudioThread.join();
1825
1826     mVideo.mEOS = true;
1827     std::unique_lock<std::mutex> lock{mVideo.mPictQMutex};
1828     while(!mVideo.mFinalUpdate)
1829         mVideo.mPictQCond.wait(lock);
1830     lock.unlock();
1831
1832     SDL_Event evt{};
1833     evt.user.type = FF_MOVIE_DONE_EVENT;
1834     SDL_PushEvent(&evt);
1835
1836     return 0;
1837 }
1838
1839 void MovieState::stop()
1840 {
1841     mQuit = true;
1842     mAudio.mQueue.flush();
1843     mVideo.mQueue.flush();
1844 }
1845
1846
1847 // Helper class+method to print the time with human-readable formatting.
1848 struct PrettyTime {
1849     seconds mTime;
1850 };
1851 std::ostream &operator<<(std::ostream &os, const PrettyTime &rhs)
1852 {
1853     using hours = std::chrono::hours;
1854     using minutes = std::chrono::minutes;
1855
1856     seconds t{rhs.mTime};
1857     if(t.count() < 0)
1858     {
1859         os << '-';
1860         t *= -1;
1861     }
1862
1863     // Only handle up to hour formatting
1864     if(t >= hours{1})
1865         os << duration_cast<hours>(t).count() << 'h' << std::setfill('0') << std::setw(2)
1866            << (duration_cast<minutes>(t).count() % 60) << 'm';
1867     else
1868         os << duration_cast<minutes>(t).count() << 'm' << std::setfill('0');
1869     os << std::setw(2) << (duration_cast<seconds>(t).count() % 60) << 's' << std::setw(0)
1870        << std::setfill(' ');
1871     return os;
1872 }
1873
1874 } // namespace
1875
1876
1877 int main(int argc, char *argv[])
1878 {
1879     std::unique_ptr<MovieState> movState;
1880
1881     if(argc < 2)
1882     {
1883         std::cerr<< "Usage: "<<argv[0]<<" [-device <device name>] [-direct] <files...>" <<std::endl;
1884         return 1;
1885     }
1886     /* Register all formats and codecs */
1887 #if !(LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 9, 100))
1888     av_register_all();
1889 #endif
1890     /* Initialize networking protocols */
1891     avformat_network_init();
1892
1893     if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS))
1894     {
1895         std::cerr<< "Could not initialize SDL - <<"<<SDL_GetError() <<std::endl;
1896         return 1;
1897     }
1898
1899     /* Make a window to put our video */
1900     SDL_Window *screen{SDL_CreateWindow(AppName.c_str(), 0, 0, 640, 480, SDL_WINDOW_RESIZABLE)};
1901     if(!screen)
1902     {
1903         std::cerr<< "SDL: could not set video mode - exiting" <<std::endl;
1904         return 1;
1905     }
1906     /* Make a renderer to handle the texture image surface and rendering. */
1907     Uint32 render_flags{SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC};
1908     SDL_Renderer *renderer{SDL_CreateRenderer(screen, -1, render_flags)};
1909     if(renderer)
1910     {
1911         SDL_RendererInfo rinf{};
1912         bool ok{false};
1913
1914         /* Make sure the renderer supports IYUV textures. If not, fallback to a
1915          * software renderer. */
1916         if(SDL_GetRendererInfo(renderer, &rinf) == 0)
1917         {
1918             for(Uint32 i{0u};!ok && i < rinf.num_texture_formats;i++)
1919                 ok = (rinf.texture_formats[i] == SDL_PIXELFORMAT_IYUV);
1920         }
1921         if(!ok)
1922         {
1923             std::cerr<< "IYUV pixelformat textures not supported on renderer "<<rinf.name <<std::endl;
1924             SDL_DestroyRenderer(renderer);
1925             renderer = nullptr;
1926         }
1927     }
1928     if(!renderer)
1929     {
1930         render_flags = SDL_RENDERER_SOFTWARE | SDL_RENDERER_PRESENTVSYNC;
1931         renderer = SDL_CreateRenderer(screen, -1, render_flags);
1932     }
1933     if(!renderer)
1934     {
1935         std::cerr<< "SDL: could not create renderer - exiting" <<std::endl;
1936         return 1;
1937     }
1938     SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
1939     SDL_RenderFillRect(renderer, nullptr);
1940     SDL_RenderPresent(renderer);
1941
1942     /* Open an audio device */
1943     ++argv; --argc;
1944     if(InitAL(&argv, &argc))
1945     {
1946         std::cerr<< "Failed to set up audio device" <<std::endl;
1947         return 1;
1948     }
1949
1950     {
1951         auto device = alcGetContextsDevice(alcGetCurrentContext());
1952         if(alcIsExtensionPresent(device, "ALC_SOFT_device_clock"))
1953         {
1954             std::cout<< "Found ALC_SOFT_device_clock" <<std::endl;
1955             alcGetInteger64vSOFT = reinterpret_cast<LPALCGETINTEGER64VSOFT>(
1956                 alcGetProcAddress(device, "alcGetInteger64vSOFT")
1957             );
1958         }
1959     }
1960
1961     if(alIsExtensionPresent("AL_SOFT_source_latency"))
1962     {
1963         std::cout<< "Found AL_SOFT_source_latency" <<std::endl;
1964         alGetSourcei64vSOFT = reinterpret_cast<LPALGETSOURCEI64VSOFT>(
1965             alGetProcAddress("alGetSourcei64vSOFT")
1966         );
1967     }
1968     if(alIsExtensionPresent("AL_SOFT_events"))
1969     {
1970         std::cout<< "Found AL_SOFT_events" <<std::endl;
1971         alEventControlSOFT = reinterpret_cast<LPALEVENTCONTROLSOFT>(
1972             alGetProcAddress("alEventControlSOFT"));
1973         alEventCallbackSOFT = reinterpret_cast<LPALEVENTCALLBACKSOFT>(
1974             alGetProcAddress("alEventCallbackSOFT"));
1975     }
1976     if(alIsExtensionPresent("AL_SOFT_callback_buffer"))
1977     {
1978         std::cout<< "Found AL_SOFT_callback_buffer" <<std::endl;
1979         alBufferCallbackSOFT = reinterpret_cast<LPALBUFFERCALLBACKSOFT>(
1980             alGetProcAddress("alBufferCallbackSOFT"));
1981     }
1982
1983     int fileidx{0};
1984     for(;fileidx < argc;++fileidx)
1985     {
1986         if(strcmp(argv[fileidx], "-direct") == 0)
1987         {
1988             if(alIsExtensionPresent("AL_SOFT_direct_channels_remix"))
1989             {
1990                 std::cout<< "Found AL_SOFT_direct_channels_remix" <<std::endl;
1991                 DirectOutMode = AL_REMIX_UNMATCHED_SOFT;
1992             }
1993             else if(alIsExtensionPresent("AL_SOFT_direct_channels"))
1994             {
1995                 std::cout<< "Found AL_SOFT_direct_channels" <<std::endl;
1996                 DirectOutMode = AL_DROP_UNMATCHED_SOFT;
1997             }
1998             else
1999                 std::cerr<< "AL_SOFT_direct_channels not supported for direct output" <<std::endl;
2000         }
2001         else if(strcmp(argv[fileidx], "-wide") == 0)
2002         {
2003             if(!alIsExtensionPresent("AL_EXT_STEREO_ANGLES"))
2004                 std::cerr<< "AL_EXT_STEREO_ANGLES not supported for wide stereo" <<std::endl;
2005             else
2006             {
2007                 std::cout<< "Found AL_EXT_STEREO_ANGLES" <<std::endl;
2008                 EnableWideStereo = true;
2009             }
2010         }
2011         else if(strcmp(argv[fileidx], "-uhj") == 0)
2012         {
2013             if(!alIsExtensionPresent("AL_SOFT_UHJ"))
2014                 std::cerr<< "AL_SOFT_UHJ not supported for UHJ decoding" <<std::endl;
2015             else
2016             {
2017                 std::cout<< "Found AL_SOFT_UHJ" <<std::endl;
2018                 EnableUhj = true;
2019             }
2020         }
2021         else if(strcmp(argv[fileidx], "-superstereo") == 0)
2022         {
2023             if(!alIsExtensionPresent("AL_SOFT_UHJ"))
2024                 std::cerr<< "AL_SOFT_UHJ not supported for Super Stereo decoding" <<std::endl;
2025             else
2026             {
2027                 std::cout<< "Found AL_SOFT_UHJ (Super Stereo)" <<std::endl;
2028                 EnableSuperStereo = true;
2029             }
2030         }
2031         else if(strcmp(argv[fileidx], "-novideo") == 0)
2032             DisableVideo = true;
2033         else
2034             break;
2035     }
2036
2037     while(fileidx < argc && !movState)
2038     {
2039         movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
2040         if(!movState->prepare()) movState = nullptr;
2041     }
2042     if(!movState)
2043     {
2044         std::cerr<< "Could not start a video" <<std::endl;
2045         return 1;
2046     }
2047     movState->setTitle(screen);
2048
2049     /* Default to going to the next movie at the end of one. */
2050     enum class EomAction {
2051         Next, Quit
2052     } eom_action{EomAction::Next};
2053     seconds last_time{seconds::min()};
2054     while(1)
2055     {
2056         /* SDL_WaitEventTimeout is broken, just force a 10ms sleep. */
2057         std::this_thread::sleep_for(milliseconds{10});
2058
2059         auto cur_time = std::chrono::duration_cast<seconds>(movState->getMasterClock());
2060         if(cur_time != last_time)
2061         {
2062             auto end_time = std::chrono::duration_cast<seconds>(movState->getDuration());
2063             std::cout<< "    \r "<<PrettyTime{cur_time}<<" / "<<PrettyTime{end_time} <<std::flush;
2064             last_time = cur_time;
2065         }
2066
2067         bool force_redraw{false};
2068         SDL_Event event{};
2069         while(SDL_PollEvent(&event) != 0)
2070         {
2071             switch(event.type)
2072             {
2073             case SDL_KEYDOWN:
2074                 switch(event.key.keysym.sym)
2075                 {
2076                 case SDLK_ESCAPE:
2077                     movState->stop();
2078                     eom_action = EomAction::Quit;
2079                     break;
2080
2081                 case SDLK_n:
2082                     movState->stop();
2083                     eom_action = EomAction::Next;
2084                     break;
2085
2086                 default:
2087                     break;
2088                 }
2089                 break;
2090
2091             case SDL_WINDOWEVENT:
2092                 switch(event.window.event)
2093                 {
2094                 case SDL_WINDOWEVENT_RESIZED:
2095                     SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
2096                     SDL_RenderFillRect(renderer, nullptr);
2097                     force_redraw = true;
2098                     break;
2099
2100                 case SDL_WINDOWEVENT_EXPOSED:
2101                     force_redraw = true;
2102                     break;
2103
2104                 default:
2105                     break;
2106                 }
2107                 break;
2108
2109             case SDL_QUIT:
2110                 movState->stop();
2111                 eom_action = EomAction::Quit;
2112                 break;
2113
2114             case FF_MOVIE_DONE_EVENT:
2115                 std::cout<<'\n';
2116                 last_time = seconds::min();
2117                 if(eom_action != EomAction::Quit)
2118                 {
2119                     movState = nullptr;
2120                     while(fileidx < argc && !movState)
2121                     {
2122                         movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
2123                         if(!movState->prepare()) movState = nullptr;
2124                     }
2125                     if(movState)
2126                     {
2127                         movState->setTitle(screen);
2128                         break;
2129                     }
2130                 }
2131
2132                 /* Nothing more to play. Shut everything down and quit. */
2133                 movState = nullptr;
2134
2135                 CloseAL();
2136
2137                 SDL_DestroyRenderer(renderer);
2138                 renderer = nullptr;
2139                 SDL_DestroyWindow(screen);
2140                 screen = nullptr;
2141
2142                 SDL_Quit();
2143                 exit(0);
2144
2145             default:
2146                 break;
2147             }
2148         }
2149
2150         movState->mVideo.updateVideo(screen, renderer, force_redraw);
2151     }
2152
2153     std::cerr<< "SDL_WaitEvent error - "<<SDL_GetError() <<std::endl;
2154     return 1;
2155 }