examples/alffplay.cpp

   1 /*
   2  * An example showing how to play a stream sync'd to video, using ffmpeg.
   3  *
   4  * Requires C++14.
   5  */
   6
   7 #include <condition_variable>
   8 #include <functional>
   9 #include <algorithm>
  10 #include <iostream>
  11 #include <utility>
  12 #include <iomanip>
  13 #include <cstdint>
  14 #include <cstring>
  15 #include <cstdlib>
  16 #include <atomic>
  17 #include <cerrno>
  18 #include <chrono>
  19 #include <cstdio>
  20 #include <memory>
  21 #include <string>
  22 #include <thread>
  23 #include <vector>
  24 #include <array>
  25 #include <cmath>
  26 #include <deque>
  27 #include <mutex>
  28 #include <ratio>
  29
  30 extern "C" {
  31 #ifdef __GNUC__
  32 _Pragma("GCC diagnostic push")
  33 _Pragma("GCC diagnostic ignored \"-Wconversion\"")
  34 _Pragma("GCC diagnostic ignored \"-Wold-style-cast\"")
  35 #endif
  36 #include "libavcodec/avcodec.h"
  37 #include "libavformat/avformat.h"
  38 #include "libavformat/avio.h"
  39 #include "libavformat/version.h"
  40 #include "libavutil/avutil.h"
  41 #include "libavutil/error.h"
  42 #include "libavutil/frame.h"
  43 #include "libavutil/mem.h"
  44 #include "libavutil/pixfmt.h"
  45 #include "libavutil/rational.h"
  46 #include "libavutil/samplefmt.h"
  47 #include "libavutil/time.h"
  48 #include "libavutil/version.h"
  49 #include "libavutil/channel_layout.h"
  50 #include "libswscale/swscale.h"
  51 #include "libswresample/swresample.h"
  52
  53 constexpr auto AVNoPtsValue = AV_NOPTS_VALUE;
  54 constexpr auto AVErrorEOF = AVERROR_EOF;
  55
  56 struct SwsContext;
  57 #ifdef __GNUC__
  58 _Pragma("GCC diagnostic pop")
  59 #endif
  60 }
  61
  62 #include "SDL.h"
  63
  64 #include "AL/alc.h"
  65 #include "AL/al.h"
  66 #include "AL/alext.h"
  67
  68 #include "common/alhelpers.h"
  69
  70 extern "C" {
  71 /* Undefine this to disable use of experimental extensions. Don't use for
  72  * production code! Interfaces and behavior may change prior to being
  73  * finalized.
  74  */
  75 #define ALLOW_EXPERIMENTAL_EXTS
  76
  77 #ifdef ALLOW_EXPERIMENTAL_EXTS
  78 #ifndef AL_SOFT_events
  79 #define AL_SOFT_events 1
  80 #define AL_EVENT_CALLBACK_FUNCTION_SOFT          0x19A2
  81 #define AL_EVENT_CALLBACK_USER_PARAM_SOFT        0x19A3
  82 #define AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT      0x19A4
  83 #define AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT  0x19A5
  84 #define AL_EVENT_TYPE_DISCONNECTED_SOFT          0x19A6
  85 typedef void (AL_APIENTRY*ALEVENTPROCSOFT)(ALenum eventType, ALuint object, ALuint param,
  86                                            ALsizei length, const ALchar *message,
  87                                            void *userParam);
  88 typedef void (AL_APIENTRY*LPALEVENTCONTROLSOFT)(ALsizei count, const ALenum *types, ALboolean enable);
  89 typedef void (AL_APIENTRY*LPALEVENTCALLBACKSOFT)(ALEVENTPROCSOFT callback, void *userParam);
  90 typedef void* (AL_APIENTRY*LPALGETPOINTERSOFT)(ALenum pname);
  91 typedef void (AL_APIENTRY*LPALGETPOINTERVSOFT)(ALenum pname, void **values);
  92 #endif
  93
  94 #ifndef AL_SOFT_callback_buffer
  95 #define AL_SOFT_callback_buffer
  96 typedef unsigned int ALbitfieldSOFT;
  97 #define AL_BUFFER_CALLBACK_FUNCTION_SOFT         0x19A0
  98 #define AL_BUFFER_CALLBACK_USER_PARAM_SOFT       0x19A1
  99 typedef ALsizei (AL_APIENTRY*LPALBUFFERCALLBACKTYPESOFT)(ALvoid *userptr, ALvoid *sampledata, ALsizei numsamples);
 100 typedef void (AL_APIENTRY*LPALBUFFERCALLBACKSOFT)(ALuint buffer, ALenum format, ALsizei freq, LPALBUFFERCALLBACKTYPESOFT callback, ALvoid *userptr, ALbitfieldSOFT flags);
 101 typedef void (AL_APIENTRY*LPALGETBUFFERPTRSOFT)(ALuint buffer, ALenum param, ALvoid **value);
 102 typedef void (AL_APIENTRY*LPALGETBUFFER3PTRSOFT)(ALuint buffer, ALenum param, ALvoid **value1, ALvoid **value2, ALvoid **value3);
 103 typedef void (AL_APIENTRY*LPALGETBUFFERPTRVSOFT)(ALuint buffer, ALenum param, ALvoid **values);
 104 #endif
 105 #endif /* ALLOW_EXPERIMENTAL_EXTS */
 106 }
 107
 108 namespace {
 109
 110 inline constexpr int64_t operator "" _i64(unsigned long long int n) noexcept { return static_cast<int64_t>(n); }
 111
 112 #ifndef M_PI
 113 #define M_PI (3.14159265358979323846)
 114 #endif
 115
 116 using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1_i64<<32)>>;
 117 using nanoseconds = std::chrono::nanoseconds;
 118 using microseconds = std::chrono::microseconds;
 119 using milliseconds = std::chrono::milliseconds;
 120 using seconds = std::chrono::seconds;
 121 using seconds_d64 = std::chrono::duration<double>;
 122 using std::chrono::duration_cast;
 123
 124 const std::string AppName{"alffplay"};
 125
 126 ALenum DirectOutMode{AL_FALSE};
 127 bool EnableWideStereo{false};
 128 bool DisableVideo{false};
 129 LPALGETSOURCEI64VSOFT alGetSourcei64vSOFT;
 130 LPALCGETINTEGER64VSOFT alcGetInteger64vSOFT;
 131
 132 #ifdef AL_SOFT_events
 133 LPALEVENTCONTROLSOFT alEventControlSOFT;
 134 LPALEVENTCALLBACKSOFT alEventCallbackSOFT;
 135 #endif
 136
 137 #ifdef AL_SOFT_callback_buffer
 138 LPALBUFFERCALLBACKSOFT alBufferCallbackSOFT;
 139 #endif
 140
 141 const seconds AVNoSyncThreshold{10};
 142
 143 #define VIDEO_PICTURE_QUEUE_SIZE 24
 144
 145 const seconds_d64 AudioSyncThreshold{0.03};
 146 const milliseconds AudioSampleCorrectionMax{50};
 147 /* Averaging filter coefficient for audio sync. */
 148 #define AUDIO_DIFF_AVG_NB 20
 149 const double AudioAvgFilterCoeff{std::pow(0.01, 1.0/AUDIO_DIFF_AVG_NB)};
 150 /* Per-buffer size, in time */
 151 constexpr milliseconds AudioBufferTime{20};
 152 /* Buffer total size, in time (should be divisible by the buffer time) */
 153 constexpr milliseconds AudioBufferTotalTime{800};
 154 constexpr auto AudioBufferCount = AudioBufferTotalTime / AudioBufferTime;
 155
 156 enum {
 157     FF_MOVIE_DONE_EVENT = SDL_USEREVENT
 158 };
 159
 160 enum class SyncMaster {
 161     Audio,
 162     Video,
 163     External,
 164
 165     Default = External
 166 };
 167
 168
 169 inline microseconds get_avtime()
 170 { return microseconds{av_gettime()}; }
 171
 172 /* Define unique_ptrs to auto-cleanup associated ffmpeg objects. */
 173 struct AVIOContextDeleter {
 174     void operator()(AVIOContext *ptr) { avio_closep(&ptr); }
 175 };
 176 using AVIOContextPtr = std::unique_ptr<AVIOContext,AVIOContextDeleter>;
 177
 178 struct AVFormatCtxDeleter {
 179     void operator()(AVFormatContext *ptr) { avformat_close_input(&ptr); }
 180 };
 181 using AVFormatCtxPtr = std::unique_ptr<AVFormatContext,AVFormatCtxDeleter>;
 182
 183 struct AVCodecCtxDeleter {
 184     void operator()(AVCodecContext *ptr) { avcodec_free_context(&ptr); }
 185 };
 186 using AVCodecCtxPtr = std::unique_ptr<AVCodecContext,AVCodecCtxDeleter>;
 187
 188 struct AVFrameDeleter {
 189     void operator()(AVFrame *ptr) { av_frame_free(&ptr); }
 190 };
 191 using AVFramePtr = std::unique_ptr<AVFrame,AVFrameDeleter>;
 192
 193 struct SwrContextDeleter {
 194     void operator()(SwrContext *ptr) { swr_free(&ptr); }
 195 };
 196 using SwrContextPtr = std::unique_ptr<SwrContext,SwrContextDeleter>;
 197
 198 struct SwsContextDeleter {
 199     void operator()(SwsContext *ptr) { sws_freeContext(ptr); }
 200 };
 201 using SwsContextPtr = std::unique_ptr<SwsContext,SwsContextDeleter>;
 202
 203
 204 template<size_t SizeLimit>
 205 class PacketQueue {
 206     std::mutex mMutex;
 207     std::condition_variable mCondVar;
 208     std::deque<AVPacket> mPackets;
 209     size_t mTotalSize{0};
 210     bool mFinished{false};
 211
 212     AVPacket *getPacket(std::unique_lock<std::mutex> &lock)
 213     {
 214         while(mPackets.empty() && !mFinished)
 215             mCondVar.wait(lock);
 216         return mPackets.empty() ? nullptr : &mPackets.front();
 217     }
 218
 219     void pop()
 220     {
 221         AVPacket *pkt = &mPackets.front();
 222         mTotalSize -= static_cast<unsigned int>(pkt->size);
 223         av_packet_unref(pkt);
 224         mPackets.pop_front();
 225     }
 226
 227 public:
 228     ~PacketQueue()
 229     {
 230         for(AVPacket &pkt : mPackets)
 231             av_packet_unref(&pkt);
 232         mPackets.clear();
 233         mTotalSize = 0;
 234     }
 235
 236     int sendTo(AVCodecContext *codecctx)
 237     {
 238         std::unique_lock<std::mutex> lock{mMutex};
 239
 240         AVPacket *pkt{getPacket(lock)};
 241         if(!pkt) return avcodec_send_packet(codecctx, nullptr);
 242
 243         const int ret{avcodec_send_packet(codecctx, pkt)};
 244         if(ret != AVERROR(EAGAIN))
 245         {
 246             if(ret < 0)
 247                 std::cerr<< "Failed to send packet: "<<ret <<std::endl;
 248             pop();
 249         }
 250         return ret;
 251     }
 252
 253     void setFinished()
 254     {
 255         {
 256             std::lock_guard<std::mutex> _{mMutex};
 257             mFinished = true;
 258         }
 259         mCondVar.notify_one();
 260     }
 261
 262     bool put(const AVPacket *pkt)
 263     {
 264         {
 265             std::unique_lock<std::mutex> lock{mMutex};
 266             if(mTotalSize >= SizeLimit)
 267                 return false;
 268
 269             mPackets.push_back(AVPacket{});
 270             if(av_packet_ref(&mPackets.back(), pkt) != 0)
 271             {
 272                 mPackets.pop_back();
 273                 return true;
 274             }
 275
 276             mTotalSize += static_cast<unsigned int>(mPackets.back().size);
 277         }
 278         mCondVar.notify_one();
 279         return true;
 280     }
 281 };
 282
 283
 284 struct MovieState;
 285
 286 struct AudioState {
 287     MovieState &mMovie;
 288
 289     AVStream *mStream{nullptr};
 290     AVCodecCtxPtr mCodecCtx;
 291
 292     PacketQueue<2*1024*1024> mPackets;
 293
 294     /* Used for clock difference average computation */
 295     seconds_d64 mClockDiffAvg{0};
 296
 297     /* Time of the next sample to be buffered */
 298     nanoseconds mCurrentPts{0};
 299
 300     /* Device clock time that the stream started at. */
 301     nanoseconds mDeviceStartTime{nanoseconds::min()};
 302
 303     /* Decompressed sample frame, and swresample context for conversion */
 304     AVFramePtr    mDecodedFrame;
 305     SwrContextPtr mSwresCtx;
 306
 307     /* Conversion format, for what gets fed to OpenAL */
 308     uint64_t       mDstChanLayout{0};
 309     AVSampleFormat mDstSampleFmt{AV_SAMPLE_FMT_NONE};
 310
 311     /* Storage of converted samples */
 312     uint8_t *mSamples{nullptr};
 313     int mSamplesLen{0}; /* In samples */
 314     int mSamplesPos{0};
 315     int mSamplesMax{0};
 316
 317     std::unique_ptr<uint8_t[]> mBufferData;
 318     size_t mBufferDataSize{0};
 319     std::atomic<size_t> mReadPos{0};
 320     std::atomic<size_t> mWritePos{0};
 321
 322     /* OpenAL format */
 323     ALenum mFormat{AL_NONE};
 324     ALuint mFrameSize{0};
 325
 326     std::mutex mSrcMutex;
 327     std::condition_variable mSrcCond;
 328     std::atomic_flag mConnected;
 329     ALuint mSource{0};
 330     std::array<ALuint,AudioBufferCount> mBuffers{};
 331     ALuint mBufferIdx{0};
 332
 333     AudioState(MovieState &movie) : mMovie(movie)
 334     { mConnected.test_and_set(std::memory_order_relaxed); }
 335     ~AudioState()
 336     {
 337         if(mSource)
 338             alDeleteSources(1, &mSource);
 339         if(mBuffers[0])
 340             alDeleteBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
 341
 342         av_freep(&mSamples);
 343     }
 344
 345 #ifdef AL_SOFT_events
 346     static void AL_APIENTRY EventCallback(ALenum eventType, ALuint object, ALuint param,
 347         ALsizei length, const ALchar *message, void *userParam);
 348 #endif
 349 #ifdef AL_SOFT_callback_buffer
 350     static ALsizei AL_APIENTRY bufferCallbackC(void *userptr, void *data, ALsizei size)
 351     { return static_cast<AudioState*>(userptr)->bufferCallback(data, size); }
 352     ALsizei bufferCallback(void *data, ALsizei size);
 353 #endif
 354
 355     nanoseconds getClockNoLock();
 356     nanoseconds getClock()
 357     {
 358         std::lock_guard<std::mutex> lock{mSrcMutex};
 359         return getClockNoLock();
 360     }
 361
 362     bool startPlayback();
 363
 364     int getSync();
 365     int decodeFrame();
 366     bool readAudio(uint8_t *samples, unsigned int length, int &sample_skip);
 367     void readAudio(int sample_skip);
 368
 369     int handler();
 370 };
 371
 372 struct VideoState {
 373     MovieState &mMovie;
 374
 375     AVStream *mStream{nullptr};
 376     AVCodecCtxPtr mCodecCtx;
 377
 378     PacketQueue<14*1024*1024> mPackets;
 379
 380     /* The pts of the currently displayed frame, and the time (av_gettime) it
 381      * was last updated - used to have running video pts
 382      */
 383     nanoseconds mDisplayPts{0};
 384     microseconds mDisplayPtsTime{microseconds::min()};
 385     std::mutex mDispPtsMutex;
 386
 387     /* Swscale context for format conversion */
 388     SwsContextPtr mSwscaleCtx;
 389
 390     struct Picture {
 391         AVFramePtr mFrame{};
 392         nanoseconds mPts{nanoseconds::min()};
 393     };
 394     std::array<Picture,VIDEO_PICTURE_QUEUE_SIZE> mPictQ;
 395     std::atomic<size_t> mPictQRead{0u}, mPictQWrite{1u};
 396     std::mutex mPictQMutex;
 397     std::condition_variable mPictQCond;
 398
 399     SDL_Texture *mImage{nullptr};
 400     int mWidth{0}, mHeight{0}; /* Logical image size (actual size may be larger) */
 401     bool mFirstUpdate{true};
 402
 403     std::atomic<bool> mEOS{false};
 404     std::atomic<bool> mFinalUpdate{false};
 405
 406     VideoState(MovieState &movie) : mMovie(movie) { }
 407     ~VideoState()
 408     {
 409         if(mImage)
 410             SDL_DestroyTexture(mImage);
 411         mImage = nullptr;
 412     }
 413
 414     nanoseconds getClock();
 415
 416     void display(SDL_Window *screen, SDL_Renderer *renderer);
 417     void updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw);
 418     int handler();
 419 };
 420
 421 struct MovieState {
 422     AVIOContextPtr mIOContext;
 423     AVFormatCtxPtr mFormatCtx;
 424
 425     SyncMaster mAVSyncType{SyncMaster::Default};
 426
 427     microseconds mClockBase{microseconds::min()};
 428
 429     std::atomic<bool> mQuit{false};
 430
 431     AudioState mAudio;
 432     VideoState mVideo;
 433
 434     std::thread mParseThread;
 435     std::thread mAudioThread;
 436     std::thread mVideoThread;
 437
 438     std::string mFilename;
 439
 440     MovieState(std::string fname)
 441       : mAudio(*this), mVideo(*this), mFilename(std::move(fname))
 442     { }
 443     ~MovieState()
 444     {
 445         mQuit = true;
 446         if(mParseThread.joinable())
 447             mParseThread.join();
 448     }
 449
 450     static int decode_interrupt_cb(void *ctx);
 451     bool prepare();
 452     void setTitle(SDL_Window *window);
 453
 454     nanoseconds getClock();
 455
 456     nanoseconds getMasterClock();
 457
 458     nanoseconds getDuration();
 459
 460     int streamComponentOpen(unsigned int stream_index);
 461     int parse_handler();
 462 };
 463
 464
 465 nanoseconds AudioState::getClockNoLock()
 466 {
 467     // The audio clock is the timestamp of the sample currently being heard.
 468     if(alcGetInteger64vSOFT)
 469     {
 470         // If device start time = min, we aren't playing yet.
 471         if(mDeviceStartTime == nanoseconds::min())
 472             return nanoseconds::zero();
 473
 474         // Get the current device clock time and latency.
 475         auto device = alcGetContextsDevice(alcGetCurrentContext());
 476         ALCint64SOFT devtimes[2]{0,0};
 477         alcGetInteger64vSOFT(device, ALC_DEVICE_CLOCK_LATENCY_SOFT, 2, devtimes);
 478         auto latency = nanoseconds{devtimes[1]};
 479         auto device_time = nanoseconds{devtimes[0]};
 480
 481         // The clock is simply the current device time relative to the recorded
 482         // start time. We can also subtract the latency to get more a accurate
 483         // position of where the audio device actually is in the output stream.
 484         return device_time - mDeviceStartTime - latency;
 485     }
 486
 487     if(mBufferDataSize > 0)
 488     {
 489         if(mDeviceStartTime == nanoseconds::min())
 490             return nanoseconds::zero();
 491
 492         /* With a callback buffer and no device clock, mDeviceStartTime is
 493          * actually the timestamp of the first sample frame played. The audio
 494          * clock, then, is that plus the current source offset.
 495          */
 496         ALint64SOFT offset[2];
 497         if(alGetSourcei64vSOFT)
 498             alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
 499         else
 500         {
 501             ALint ioffset;
 502             alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
 503             offset[0] = ALint64SOFT{ioffset} << 32;
 504             offset[1] = 0;
 505         }
 506         /* NOTE: The source state must be checked last, in case an underrun
 507          * occurs and the source stops between getting the state and retrieving
 508          * the offset+latency.
 509          */
 510         ALint status;
 511         alGetSourcei(mSource, AL_SOURCE_STATE, &status);
 512
 513         nanoseconds pts{};
 514         if(status == AL_PLAYING || status == AL_PAUSED)
 515             pts = mDeviceStartTime - nanoseconds{offset[1]} +
 516                 duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
 517         else
 518         {
 519             /* If the source is stopped, the pts of the next sample to be heard
 520              * is the pts of the next sample to be buffered, minus the amount
 521              * already in the buffer ready to play.
 522              */
 523             const size_t woffset{mWritePos.load(std::memory_order_acquire)};
 524             const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
 525             const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
 526                 roffset};
 527
 528             pts = mCurrentPts - nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
 529         }
 530
 531         return pts;
 532     }
 533
 534     /* The source-based clock is based on 4 components:
 535      * 1 - The timestamp of the next sample to buffer (mCurrentPts)
 536      * 2 - The length of the source's buffer queue
 537      *     (AudioBufferTime*AL_BUFFERS_QUEUED)
 538      * 3 - The offset OpenAL is currently at in the source (the first value
 539      *     from AL_SAMPLE_OFFSET_LATENCY_SOFT)
 540      * 4 - The latency between OpenAL and the DAC (the second value from
 541      *     AL_SAMPLE_OFFSET_LATENCY_SOFT)
 542      *
 543      * Subtracting the length of the source queue from the next sample's
 544      * timestamp gives the timestamp of the sample at the start of the source
 545      * queue. Adding the source offset to that results in the timestamp for the
 546      * sample at OpenAL's current position, and subtracting the source latency
 547      * from that gives the timestamp of the sample currently at the DAC.
 548      */
 549     nanoseconds pts{mCurrentPts};
 550     if(mSource)
 551     {
 552         ALint64SOFT offset[2];
 553         if(alGetSourcei64vSOFT)
 554             alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
 555         else
 556         {
 557             ALint ioffset;
 558             alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
 559             offset[0] = ALint64SOFT{ioffset} << 32;
 560             offset[1] = 0;
 561         }
 562         ALint queued, status;
 563         alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
 564         alGetSourcei(mSource, AL_SOURCE_STATE, &status);
 565
 566         /* If the source is AL_STOPPED, then there was an underrun and all
 567          * buffers are processed, so ignore the source queue. The audio thread
 568          * will put the source into an AL_INITIAL state and clear the queue
 569          * when it starts recovery.
 570          */
 571         if(status != AL_STOPPED)
 572         {
 573             pts -= AudioBufferTime*queued;
 574             pts += duration_cast<nanoseconds>(fixed32{offset[0] / mCodecCtx->sample_rate});
 575         }
 576         /* Don't offset by the latency if the source isn't playing. */
 577         if(status == AL_PLAYING)
 578             pts -= nanoseconds{offset[1]};
 579     }
 580
 581     return std::max(pts, nanoseconds::zero());
 582 }
 583
 584 bool AudioState::startPlayback()
 585 {
 586     const size_t woffset{mWritePos.load(std::memory_order_acquire)};
 587     const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
 588     const size_t readable{((woffset >= roffset) ? woffset : (mBufferDataSize+woffset)) -
 589         roffset};
 590
 591     if(mBufferDataSize > 0)
 592     {
 593         if(readable == 0)
 594             return false;
 595         if(!alcGetInteger64vSOFT)
 596             mDeviceStartTime = mCurrentPts -
 597                 nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate;
 598     }
 599     else
 600     {
 601         ALint queued{};
 602         alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
 603         if(queued == 0) return false;
 604     }
 605
 606     alSourcePlay(mSource);
 607     if(alcGetInteger64vSOFT)
 608     {
 609         /* Subtract the total buffer queue time from the current pts to get the
 610          * pts of the start of the queue.
 611          */
 612         int64_t srctimes[2]{0,0};
 613         alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_CLOCK_SOFT, srctimes);
 614         auto device_time = nanoseconds{srctimes[1]};
 615         auto src_offset = duration_cast<nanoseconds>(fixed32{srctimes[0]}) /
 616             mCodecCtx->sample_rate;
 617
 618         /* The mixer may have ticked and incremented the device time and sample
 619          * offset, so subtract the source offset from the device time to get
 620          * the device time the source started at. Also subtract startpts to get
 621          * the device time the stream would have started at to reach where it
 622          * is now.
 623          */
 624         if(mBufferDataSize > 0)
 625         {
 626             nanoseconds startpts{mCurrentPts -
 627                 nanoseconds{seconds{readable/mFrameSize}}/mCodecCtx->sample_rate};
 628             mDeviceStartTime = device_time - src_offset - startpts;
 629         }
 630         else
 631         {
 632             nanoseconds startpts{mCurrentPts - AudioBufferTotalTime};
 633             mDeviceStartTime = device_time - src_offset - startpts;
 634         }
 635     }
 636     return true;
 637 }
 638
 639 int AudioState::getSync()
 640 {
 641     if(mMovie.mAVSyncType == SyncMaster::Audio)
 642         return 0;
 643
 644     auto ref_clock = mMovie.getMasterClock();
 645     auto diff = ref_clock - getClockNoLock();
 646
 647     if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
 648     {
 649         /* Difference is TOO big; reset accumulated average */
 650         mClockDiffAvg = seconds_d64::zero();
 651         return 0;
 652     }
 653
 654     /* Accumulate the diffs */
 655     mClockDiffAvg = mClockDiffAvg*AudioAvgFilterCoeff + diff;
 656     auto avg_diff = mClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
 657     if(avg_diff < AudioSyncThreshold/2.0 && avg_diff > -AudioSyncThreshold)
 658         return 0;
 659
 660     /* Constrain the per-update difference to avoid exceedingly large skips */
 661     diff = std::min<nanoseconds>(diff, AudioSampleCorrectionMax);
 662     return static_cast<int>(duration_cast<seconds>(diff*mCodecCtx->sample_rate).count());
 663 }
 664
 665 int AudioState::decodeFrame()
 666 {
 667     while(!mMovie.mQuit.load(std::memory_order_relaxed))
 668     {
 669         int ret;
 670         while((ret=avcodec_receive_frame(mCodecCtx.get(), mDecodedFrame.get())) == AVERROR(EAGAIN))
 671             mPackets.sendTo(mCodecCtx.get());
 672         if(ret != 0)
 673         {
 674             if(ret == AVErrorEOF) break;
 675             std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
 676             continue;
 677         }
 678
 679         if(mDecodedFrame->nb_samples <= 0)
 680             continue;
 681
 682         /* If provided, update w/ pts */
 683         if(mDecodedFrame->best_effort_timestamp != AVNoPtsValue)
 684             mCurrentPts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
 685                 static_cast<double>(mDecodedFrame->best_effort_timestamp)});
 686
 687         if(mDecodedFrame->nb_samples > mSamplesMax)
 688         {
 689             av_freep(&mSamples);
 690             av_samples_alloc(&mSamples, nullptr, mCodecCtx->channels, mDecodedFrame->nb_samples,
 691                 mDstSampleFmt, 0);
 692             mSamplesMax = mDecodedFrame->nb_samples;
 693         }
 694         /* Return the amount of sample frames converted */
 695         int data_size{swr_convert(mSwresCtx.get(), &mSamples, mDecodedFrame->nb_samples,
 696             const_cast<const uint8_t**>(mDecodedFrame->data), mDecodedFrame->nb_samples)};
 697
 698         av_frame_unref(mDecodedFrame.get());
 699         return data_size;
 700     }
 701
 702     return 0;
 703 }
 704
 705 /* Duplicates the sample at in to out, count times. The frame size is a
 706  * multiple of the template type size.
 707  */
 708 template<typename T>
 709 static void sample_dup(uint8_t *out, const uint8_t *in, size_t count, size_t frame_size)
 710 {
 711     auto *sample = reinterpret_cast<const T*>(in);
 712     auto *dst = reinterpret_cast<T*>(out);
 713     if(frame_size == sizeof(T))
 714         std::fill_n(dst, count, *sample);
 715     else
 716     {
 717         /* NOTE: frame_size is a multiple of sizeof(T). */
 718         size_t type_mult{frame_size / sizeof(T)};
 719         size_t i{0};
 720         std::generate_n(dst, count*type_mult,
 721             [sample,type_mult,&i]() -> T
 722             {
 723                 T ret = sample[i];
 724                 i = (i+1)%type_mult;
 725                 return ret;
 726             }
 727         );
 728     }
 729 }
 730
 731
 732 bool AudioState::readAudio(uint8_t *samples, unsigned int length, int &sample_skip)
 733 {
 734     unsigned int audio_size{0};
 735
 736     /* Read the next chunk of data, refill the buffer, and queue it
 737      * on the source */
 738     length /= mFrameSize;
 739     while(mSamplesLen > 0 && audio_size < length)
 740     {
 741         unsigned int rem{length - audio_size};
 742         if(mSamplesPos >= 0)
 743         {
 744             const auto len = static_cast<unsigned int>(mSamplesLen - mSamplesPos);
 745             if(rem > len) rem = len;
 746             std::copy_n(mSamples + static_cast<unsigned int>(mSamplesPos)*mFrameSize,
 747                 rem*mFrameSize, samples);
 748         }
 749         else
 750         {
 751             rem = std::min(rem, static_cast<unsigned int>(-mSamplesPos));
 752
 753             /* Add samples by copying the first sample */
 754             if((mFrameSize&7) == 0)
 755                 sample_dup<uint64_t>(samples, mSamples, rem, mFrameSize);
 756             else if((mFrameSize&3) == 0)
 757                 sample_dup<uint32_t>(samples, mSamples, rem, mFrameSize);
 758             else if((mFrameSize&1) == 0)
 759                 sample_dup<uint16_t>(samples, mSamples, rem, mFrameSize);
 760             else
 761                 sample_dup<uint8_t>(samples, mSamples, rem, mFrameSize);
 762         }
 763
 764         mSamplesPos += rem;
 765         mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 766         samples += rem*mFrameSize;
 767         audio_size += rem;
 768
 769         while(mSamplesPos >= mSamplesLen)
 770         {
 771             int frame_len = decodeFrame();
 772             if(frame_len <= 0) break;
 773
 774             mSamplesLen = frame_len;
 775             mSamplesPos = std::min(mSamplesLen, sample_skip);
 776             sample_skip -= mSamplesPos;
 777
 778             // Adjust the device start time and current pts by the amount we're
 779             // skipping/duplicating, so that the clock remains correct for the
 780             // current stream position.
 781             auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
 782             mDeviceStartTime -= skip;
 783             mCurrentPts += skip;
 784             continue;
 785         }
 786     }
 787     if(audio_size <= 0)
 788         return false;
 789
 790     if(audio_size < length)
 791     {
 792         const unsigned int rem{length - audio_size};
 793         std::fill_n(samples, rem*mFrameSize,
 794             (mDstSampleFmt == AV_SAMPLE_FMT_U8) ? 0x80 : 0x00);
 795         mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 796         audio_size += rem;
 797     }
 798     return true;
 799 }
 800
 801 void AudioState::readAudio(int sample_skip)
 802 {
 803     size_t woffset{mWritePos.load(std::memory_order_acquire)};
 804     while(mSamplesLen > 0)
 805     {
 806         const size_t roffset{mReadPos.load(std::memory_order_relaxed)};
 807
 808         if(mSamplesPos < 0)
 809         {
 810             size_t rem{(((roffset > woffset) ? roffset-1
 811                 : ((roffset == 0) ? mBufferDataSize-1
 812                 : mBufferDataSize)) - woffset) / mFrameSize};
 813             rem = std::min<size_t>(rem, static_cast<ALuint>(-mSamplesPos));
 814             if(rem == 0) break;
 815
 816             auto *splout{&mBufferData[woffset]};
 817             if((mFrameSize&7) == 0)
 818                 sample_dup<uint64_t>(splout, mSamples, rem, mFrameSize);
 819             else if((mFrameSize&3) == 0)
 820                 sample_dup<uint32_t>(splout, mSamples, rem, mFrameSize);
 821             else if((mFrameSize&1) == 0)
 822                 sample_dup<uint16_t>(splout, mSamples, rem, mFrameSize);
 823             else
 824                 sample_dup<uint8_t>(splout, mSamples, rem, mFrameSize);
 825             woffset += rem * mFrameSize;
 826             if(woffset == mBufferDataSize)
 827                 woffset = 0;
 828             mWritePos.store(woffset, std::memory_order_release);
 829             mSamplesPos += static_cast<int>(rem);
 830             mCurrentPts += nanoseconds{seconds{rem}} / mCodecCtx->sample_rate;
 831             continue;
 832         }
 833
 834         const size_t boffset{static_cast<ALuint>(mSamplesPos) * size_t{mFrameSize}};
 835         const size_t nbytes{static_cast<ALuint>(mSamplesLen)*size_t{mFrameSize} -
 836             boffset};
 837         if(roffset > woffset)
 838         {
 839             const size_t writable{roffset-woffset-1};
 840             if(writable < nbytes) break;
 841
 842             memcpy(&mBufferData[woffset], mSamples+boffset, nbytes);
 843             woffset += nbytes;
 844         }
 845         else
 846         {
 847             const size_t writable{mBufferDataSize+roffset-woffset-1};
 848             if(writable < nbytes) break;
 849
 850             const size_t todo1{std::min<size_t>(nbytes, mBufferDataSize-woffset)};
 851             const size_t todo2{nbytes - todo1};
 852
 853             memcpy(&mBufferData[woffset], mSamples+boffset, todo1);
 854             woffset += todo1;
 855             if(woffset == mBufferDataSize)
 856             {
 857                 woffset = 0;
 858                 if(todo2 > 0)
 859                 {
 860                     memcpy(&mBufferData[woffset], mSamples+boffset+todo1, todo2);
 861                     woffset += todo2;
 862                 }
 863             }
 864         }
 865         mWritePos.store(woffset, std::memory_order_release);
 866         mCurrentPts += nanoseconds{seconds{mSamplesLen-mSamplesPos}} / mCodecCtx->sample_rate;
 867
 868         do {
 869             mSamplesLen = decodeFrame();
 870             if(mSamplesLen <= 0) break;
 871
 872             mSamplesPos = std::min(mSamplesLen, sample_skip);
 873             sample_skip -= mSamplesPos;
 874
 875             auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
 876             mDeviceStartTime -= skip;
 877             mCurrentPts += skip;
 878         } while(mSamplesPos >= mSamplesLen);
 879     }
 880 }
 881
 882
 883 #ifdef AL_SOFT_events
 884 void AL_APIENTRY AudioState::EventCallback(ALenum eventType, ALuint object, ALuint param,
 885     ALsizei length, const ALchar *message, void *userParam)
 886 {
 887     auto self = static_cast<AudioState*>(userParam);
 888
 889     if(eventType == AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT)
 890     {
 891         /* Temporarily lock the source mutex to ensure it's not between
 892          * checking the processed count and going to sleep.
 893          */
 894         std::unique_lock<std::mutex>{self->mSrcMutex}.unlock();
 895         self->mSrcCond.notify_one();
 896         return;
 897     }
 898
 899     std::cout<< "\n---- AL Event on AudioState "<<self<<" ----\nEvent: ";
 900     switch(eventType)
 901     {
 902     case AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT: std::cout<< "Buffer completed"; break;
 903     case AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT: std::cout<< "Source state changed"; break;
 904     case AL_EVENT_TYPE_DISCONNECTED_SOFT: std::cout<< "Disconnected"; break;
 905     default:
 906         std::cout<< "0x"<<std::hex<<std::setw(4)<<std::setfill('0')<<eventType<<std::dec<<
 907             std::setw(0)<<std::setfill(' '); break;
 908     }
 909     std::cout<< "\n"
 910         "Object ID: "<<object<<"\n"
 911         "Parameter: "<<param<<"\n"
 912         "Message: "<<std::string{message, static_cast<ALuint>(length)}<<"\n----"<<
 913         std::endl;
 914
 915     if(eventType == AL_EVENT_TYPE_DISCONNECTED_SOFT)
 916     {
 917         {
 918             std::lock_guard<std::mutex> lock{self->mSrcMutex};
 919             self->mConnected.clear(std::memory_order_release);
 920         }
 921         self->mSrcCond.notify_one();
 922     }
 923 }
 924 #endif
 925
 926 #ifdef AL_SOFT_callback_buffer
 927 ALsizei AudioState::bufferCallback(void *data, ALsizei size)
 928 {
 929     ALsizei got{0};
 930
 931     size_t roffset{mReadPos.load(std::memory_order_acquire)};
 932     while(got < size)
 933     {
 934         const size_t woffset{mWritePos.load(std::memory_order_relaxed)};
 935         if(woffset == roffset) break;
 936
 937         size_t todo{((woffset < roffset) ? mBufferDataSize : woffset) - roffset};
 938         todo = std::min<size_t>(todo, static_cast<ALuint>(size-got));
 939
 940         memcpy(data, &mBufferData[roffset], todo);
 941         data = static_cast<ALbyte*>(data) + todo;
 942         got += static_cast<ALsizei>(todo);
 943
 944         roffset += todo;
 945         if(roffset == mBufferDataSize)
 946             roffset = 0;
 947     }
 948     mReadPos.store(roffset, std::memory_order_release);
 949
 950     return got;
 951 }
 952 #endif
 953
 954 int AudioState::handler()
 955 {
 956     std::unique_lock<std::mutex> srclock{mSrcMutex, std::defer_lock};
 957     milliseconds sleep_time{AudioBufferTime / 3};
 958     ALenum fmt;
 959
 960 #ifdef AL_SOFT_events
 961     const std::array<ALenum,3> evt_types{{
 962         AL_EVENT_TYPE_BUFFER_COMPLETED_SOFT, AL_EVENT_TYPE_SOURCE_STATE_CHANGED_SOFT,
 963         AL_EVENT_TYPE_DISCONNECTED_SOFT}};
 964     if(alEventControlSOFT)
 965     {
 966         alEventControlSOFT(evt_types.size(), evt_types.data(), AL_TRUE);
 967         alEventCallbackSOFT(EventCallback, this);
 968         sleep_time = AudioBufferTotalTime;
 969     }
 970 #endif
 971 #ifdef AL_SOFT_bformat_ex
 972     const bool has_bfmt_ex{alIsExtensionPresent("AL_SOFT_bformat_ex") != AL_FALSE};
 973     ALenum ambi_layout{AL_FUMA_SOFT};
 974     ALenum ambi_scale{AL_FUMA_SOFT};
 975 #endif
 976
 977     /* Find a suitable format for OpenAL. */
 978     mDstChanLayout = 0;
 979     mFormat = AL_NONE;
 980     if((mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLT || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLTP) &&
 981        alIsExtensionPresent("AL_EXT_FLOAT32"))
 982     {
 983         mDstSampleFmt = AV_SAMPLE_FMT_FLT;
 984         mFrameSize = 4;
 985         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1 &&
 986            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 987            (fmt=alGetEnumValue("AL_FORMAT_71CHN32")) != AL_NONE && fmt != -1)
 988         {
 989             mDstChanLayout = mCodecCtx->channel_layout;
 990             mFrameSize *= 8;
 991             mFormat = fmt;
 992         }
 993         if((mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1 ||
 994             mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK) &&
 995            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 996            (fmt=alGetEnumValue("AL_FORMAT_51CHN32")) != AL_NONE && fmt != -1)
 997         {
 998             mDstChanLayout = mCodecCtx->channel_layout;
 999             mFrameSize *= 6;
1000             mFormat = fmt;
1001         }
1002         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
1003         {
1004             mDstChanLayout = mCodecCtx->channel_layout;
1005             mFrameSize *= 1;
1006             mFormat = AL_FORMAT_MONO_FLOAT32;
1007         }
1008         /* Assume 3D B-Format (ambisonics) if the channel layout is blank and
1009          * there's 4 or more channels. FFmpeg/libavcodec otherwise seems to
1010          * have no way to specify if the source is actually B-Format (let alone
1011          * if it's 2D or 3D).
1012          */
1013         if(mCodecCtx->channel_layout == 0 && mCodecCtx->channels >= 4 &&
1014            alIsExtensionPresent("AL_EXT_BFORMAT") &&
1015            (fmt=alGetEnumValue("AL_FORMAT_BFORMAT3D_FLOAT32")) != AL_NONE && fmt != -1)
1016         {
1017             int order{static_cast<int>(std::sqrt(mCodecCtx->channels)) - 1};
1018             if((order+1)*(order+1) == mCodecCtx->channels ||
1019                (order+1)*(order+1) + 2 == mCodecCtx->channels)
1020             {
1021                 /* OpenAL only supports first-order with AL_EXT_BFORMAT, which
1022                  * is 4 channels for 3D buffers.
1023                  */
1024                 mFrameSize *= 4;
1025                 mFormat = fmt;
1026             }
1027         }
1028         if(!mFormat)
1029         {
1030             mDstChanLayout = AV_CH_LAYOUT_STEREO;
1031             mFrameSize *= 2;
1032             mFormat = AL_FORMAT_STEREO_FLOAT32;
1033         }
1034     }
1035     if(mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8P)
1036     {
1037         mDstSampleFmt = AV_SAMPLE_FMT_U8;
1038         mFrameSize = 1;
1039         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1 &&
1040            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
1041            (fmt=alGetEnumValue("AL_FORMAT_71CHN8")) != AL_NONE && fmt != -1)
1042         {
1043             mDstChanLayout = mCodecCtx->channel_layout;
1044             mFrameSize *= 8;
1045             mFormat = fmt;
1046         }
1047         if((mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1 ||
1048             mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK) &&
1049            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
1050            (fmt=alGetEnumValue("AL_FORMAT_51CHN8")) != AL_NONE && fmt != -1)
1051         {
1052             mDstChanLayout = mCodecCtx->channel_layout;
1053             mFrameSize *= 6;
1054             mFormat = fmt;
1055         }
1056         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
1057         {
1058             mDstChanLayout = mCodecCtx->channel_layout;
1059             mFrameSize *= 1;
1060             mFormat = AL_FORMAT_MONO8;
1061         }
1062         if(mCodecCtx->channel_layout == 0 && mCodecCtx->channels >= 4 &&
1063            alIsExtensionPresent("AL_EXT_BFORMAT") &&
1064            (fmt=alGetEnumValue("AL_FORMAT_BFORMAT3D8")) != AL_NONE && fmt != -1)
1065         {
1066             int order{static_cast<int>(std::sqrt(mCodecCtx->channels)) - 1};
1067             if((order+1)*(order+1) == mCodecCtx->channels ||
1068                (order+1)*(order+1) + 2 == mCodecCtx->channels)
1069             {
1070                 mFrameSize *= 4;
1071                 mFormat = fmt;
1072             }
1073         }
1074         if(!mFormat)
1075         {
1076             mDstChanLayout = AV_CH_LAYOUT_STEREO;
1077             mFrameSize *= 2;
1078             mFormat = AL_FORMAT_STEREO8;
1079         }
1080     }
1081     if(!mFormat)
1082     {
1083         mDstSampleFmt = AV_SAMPLE_FMT_S16;
1084         mFrameSize = 2;
1085         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1 &&
1086            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
1087            (fmt=alGetEnumValue("AL_FORMAT_71CHN16")) != AL_NONE && fmt != -1)
1088         {
1089             mDstChanLayout = mCodecCtx->channel_layout;
1090             mFrameSize *= 8;
1091             mFormat = fmt;
1092         }
1093         if((mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1 ||
1094             mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK) &&
1095            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
1096            (fmt=alGetEnumValue("AL_FORMAT_51CHN16")) != AL_NONE && fmt != -1)
1097         {
1098             mDstChanLayout = mCodecCtx->channel_layout;
1099             mFrameSize *= 6;
1100             mFormat = fmt;
1101         }
1102         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
1103         {
1104             mDstChanLayout = mCodecCtx->channel_layout;
1105             mFrameSize *= 1;
1106             mFormat = AL_FORMAT_MONO16;
1107         }
1108         if(mCodecCtx->channel_layout == 0 && mCodecCtx->channels >= 4 &&
1109            alIsExtensionPresent("AL_EXT_BFORMAT") &&
1110            (fmt=alGetEnumValue("AL_FORMAT_BFORMAT3D16")) != AL_NONE && fmt != -1)
1111         {
1112             int order{static_cast<int>(std::sqrt(mCodecCtx->channels)) - 1};
1113             if((order+1)*(order+1) == mCodecCtx->channels ||
1114                (order+1)*(order+1) + 2 == mCodecCtx->channels)
1115             {
1116                 mFrameSize *= 4;
1117                 mFormat = fmt;
1118             }
1119         }
1120         if(!mFormat)
1121         {
1122             mDstChanLayout = AV_CH_LAYOUT_STEREO;
1123             mFrameSize *= 2;
1124             mFormat = AL_FORMAT_STEREO16;
1125         }
1126     }
1127     void *samples{nullptr};
1128     ALsizei buffer_len{0};
1129
1130     mSamples = nullptr;
1131     mSamplesMax = 0;
1132     mSamplesPos = 0;
1133     mSamplesLen = 0;
1134
1135     mDecodedFrame.reset(av_frame_alloc());
1136     if(!mDecodedFrame)
1137     {
1138         std::cerr<< "Failed to allocate audio frame" <<std::endl;
1139         goto finish;
1140     }
1141
1142     if(!mDstChanLayout)
1143     {
1144         /* OpenAL only supports first-order ambisonics with AL_EXT_BFORMAT, so
1145          * we have to drop any extra channels.
1146          */
1147         mSwresCtx.reset(swr_alloc_set_opts(nullptr,
1148             (1_i64<<4)-1, mDstSampleFmt, mCodecCtx->sample_rate,
1149             (1_i64<<mCodecCtx->channels)-1, mCodecCtx->sample_fmt, mCodecCtx->sample_rate,
1150             0, nullptr));
1151
1152         /* Note that ffmpeg/libavcodec has no method to check the ambisonic
1153          * channel order and normalization, so we can only assume AmbiX as the
1154          * defacto-standard. This is not true for .amb files, which use FuMa.
1155          */
1156         std::vector<double> mtx(64*64, 0.0);
1157 #ifdef AL_SOFT_bformat_ex
1158         ambi_layout = AL_ACN_SOFT;
1159         ambi_scale = AL_SN3D_SOFT;
1160         if(has_bfmt_ex)
1161         {
1162             /* An identity matrix that doesn't remix any channels. */
1163             std::cout<< "Found AL_SOFT_bformat_ex" <<std::endl;
1164             mtx[0 + 0*64] = 1.0;
1165             mtx[1 + 1*64] = 1.0;
1166             mtx[2 + 2*64] = 1.0;
1167             mtx[3 + 3*64] = 1.0;
1168         }
1169         else
1170 #endif
1171         {
1172             std::cout<< "Found AL_EXT_BFORMAT" <<std::endl;
1173             /* Without AL_SOFT_bformat_ex, OpenAL only supports FuMa channel
1174              * ordering and normalization, so a custom matrix is needed to
1175              * scale and reorder the source from AmbiX.
1176              */
1177             mtx[0 + 0*64] = std::sqrt(0.5);
1178             mtx[3 + 1*64] = 1.0;
1179             mtx[1 + 2*64] = 1.0;
1180             mtx[2 + 3*64] = 1.0;
1181         }
1182         swr_set_matrix(mSwresCtx.get(), mtx.data(), 64);
1183     }
1184     else
1185         mSwresCtx.reset(swr_alloc_set_opts(nullptr,
1186             static_cast<int64_t>(mDstChanLayout), mDstSampleFmt, mCodecCtx->sample_rate,
1187             mCodecCtx->channel_layout ? static_cast<int64_t>(mCodecCtx->channel_layout)
1188                 : av_get_default_channel_layout(mCodecCtx->channels),
1189             mCodecCtx->sample_fmt, mCodecCtx->sample_rate,
1190             0, nullptr));
1191     if(!mSwresCtx || swr_init(mSwresCtx.get()) != 0)
1192     {
1193         std::cerr<< "Failed to initialize audio converter" <<std::endl;
1194         goto finish;
1195     }
1196
1197     alGenBuffers(static_cast<ALsizei>(mBuffers.size()), mBuffers.data());
1198     alGenSources(1, &mSource);
1199
1200     if(DirectOutMode)
1201         alSourcei(mSource, AL_DIRECT_CHANNELS_SOFT, DirectOutMode);
1202     if(EnableWideStereo)
1203     {
1204         const float angles[2]{static_cast<float>(M_PI / 3.0), static_cast<float>(-M_PI / 3.0)};
1205         alSourcefv(mSource, AL_STEREO_ANGLES, angles);
1206     }
1207 #ifdef AL_SOFT_bformat_ex
1208     if(has_bfmt_ex)
1209     {
1210         for(ALuint bufid : mBuffers)
1211         {
1212             alBufferi(bufid, AL_AMBISONIC_LAYOUT_SOFT, ambi_layout);
1213             alBufferi(bufid, AL_AMBISONIC_SCALING_SOFT, ambi_scale);
1214         }
1215     }
1216 #endif
1217
1218     if(alGetError() != AL_NO_ERROR)
1219         goto finish;
1220
1221 #ifdef AL_SOFT_callback_buffer
1222     if(alBufferCallbackSOFT)
1223     {
1224         alBufferCallbackSOFT(mBuffers[0], mFormat, mCodecCtx->sample_rate, bufferCallbackC, this,
1225             0);
1226         alSourcei(mSource, AL_BUFFER, static_cast<ALint>(mBuffers[0]));
1227         if(alGetError() != AL_NO_ERROR)
1228         {
1229             fprintf(stderr, "Failed to set buffer callback\n");
1230             alSourcei(mSource, AL_BUFFER, 0);
1231             buffer_len = static_cast<int>(duration_cast<seconds>(mCodecCtx->sample_rate *
1232                 AudioBufferTime).count() * mFrameSize);
1233         }
1234         else
1235         {
1236             mBufferDataSize = static_cast<size_t>(duration_cast<seconds>(mCodecCtx->sample_rate *
1237                 AudioBufferTotalTime).count()) * mFrameSize;
1238             mBufferData.reset(new uint8_t[mBufferDataSize]);
1239             mReadPos.store(0, std::memory_order_relaxed);
1240             mWritePos.store(0, std::memory_order_relaxed);
1241
1242             ALCint refresh{};
1243             alcGetIntegerv(alcGetContextsDevice(alcGetCurrentContext()), ALC_REFRESH, 1, &refresh);
1244             sleep_time = milliseconds{seconds{1}} / refresh;
1245         }
1246     }
1247     else
1248 #endif
1249         buffer_len = static_cast<int>(duration_cast<seconds>(mCodecCtx->sample_rate *
1250             AudioBufferTime).count() * mFrameSize);
1251     if(buffer_len > 0)
1252         samples = av_malloc(static_cast<ALuint>(buffer_len));
1253
1254     /* Prefill the codec buffer. */
1255     do {
1256         const int ret{mPackets.sendTo(mCodecCtx.get())};
1257         if(ret == AVERROR(EAGAIN) || ret == AVErrorEOF)
1258             break;
1259     } while(1);
1260
1261     srclock.lock();
1262     if(alcGetInteger64vSOFT)
1263     {
1264         int64_t devtime{};
1265         alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()), ALC_DEVICE_CLOCK_SOFT,
1266             1, &devtime);
1267         mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
1268     }
1269
1270     mSamplesLen = decodeFrame();
1271     if(mSamplesLen > 0)
1272     {
1273         mSamplesPos = std::min(mSamplesLen, getSync());
1274
1275         auto skip = nanoseconds{seconds{mSamplesPos}} / mCodecCtx->sample_rate;
1276         mDeviceStartTime -= skip;
1277         mCurrentPts += skip;
1278     }
1279
1280     while(!mMovie.mQuit.load(std::memory_order_relaxed)
1281         && mConnected.test_and_set(std::memory_order_relaxed))
1282     {
1283         ALenum state;
1284         if(mBufferDataSize > 0)
1285         {
1286             alGetSourcei(mSource, AL_SOURCE_STATE, &state);
1287             readAudio(getSync());
1288         }
1289         else
1290         {
1291             ALint processed, queued;
1292
1293             /* First remove any processed buffers. */
1294             alGetSourcei(mSource, AL_BUFFERS_PROCESSED, &processed);
1295             while(processed > 0)
1296             {
1297                 ALuint bid;
1298                 alSourceUnqueueBuffers(mSource, 1, &bid);
1299                 --processed;
1300             }
1301
1302             /* Refill the buffer queue. */
1303             int sync_skip{getSync()};
1304             alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
1305             while(static_cast<ALuint>(queued) < mBuffers.size())
1306             {
1307                 /* Read the next chunk of data, filling the buffer, and queue
1308                  * it on the source.
1309                  */
1310                 const bool got_audio{readAudio(static_cast<uint8_t*>(samples),
1311                     static_cast<ALuint>(buffer_len), sync_skip)};
1312                 if(!got_audio) break;
1313
1314                 const ALuint bufid{mBuffers[mBufferIdx]};
1315                 mBufferIdx = static_cast<ALuint>((mBufferIdx+1) % mBuffers.size());
1316
1317                 alBufferData(bufid, mFormat, samples, buffer_len, mCodecCtx->sample_rate);
1318                 alSourceQueueBuffers(mSource, 1, &bufid);
1319                 ++queued;
1320             }
1321
1322             /* Check that the source is playing. */
1323             alGetSourcei(mSource, AL_SOURCE_STATE, &state);
1324             if(state == AL_STOPPED)
1325             {
1326                 /* AL_STOPPED means there was an underrun. Clear the buffer
1327                  * queue since this likely means we're late, and rewind the
1328                  * source to get it back into an AL_INITIAL state.
1329                  */
1330                 alSourceRewind(mSource);
1331                 alSourcei(mSource, AL_BUFFER, 0);
1332                 if(alcGetInteger64vSOFT)
1333                 {
1334                     /* Also update the device start time with the current
1335                      * device clock, so the decoder knows we're running behind.
1336                      */
1337                     int64_t devtime{};
1338                     alcGetInteger64vSOFT(alcGetContextsDevice(alcGetCurrentContext()),
1339                         ALC_DEVICE_CLOCK_SOFT, 1, &devtime);
1340                     mDeviceStartTime = nanoseconds{devtime} - mCurrentPts;
1341                 }
1342                 continue;
1343             }
1344         }
1345
1346         /* (re)start the source if needed, and wait for a buffer to finish */
1347         if(state != AL_PLAYING && state != AL_PAUSED)
1348         {
1349             if(!startPlayback())
1350                 break;
1351         }
1352         if(alGetError() != AL_NO_ERROR)
1353             return false;
1354
1355         mSrcCond.wait_for(srclock, sleep_time);
1356     }
1357
1358     alSourceRewind(mSource);
1359     alSourcei(mSource, AL_BUFFER, 0);
1360     srclock.unlock();
1361
1362 finish:
1363     av_freep(&samples);
1364
1365 #ifdef AL_SOFT_events
1366     if(alEventControlSOFT)
1367     {
1368         alEventControlSOFT(evt_types.size(), evt_types.data(), AL_FALSE);
1369         alEventCallbackSOFT(nullptr, nullptr);
1370     }
1371 #endif
1372
1373     return 0;
1374 }
1375
1376
1377 nanoseconds VideoState::getClock()
1378 {
1379     /* NOTE: This returns incorrect times while not playing. */
1380     std::lock_guard<std::mutex> _{mDispPtsMutex};
1381     if(mDisplayPtsTime == microseconds::min())
1382         return nanoseconds::zero();
1383     auto delta = get_avtime() - mDisplayPtsTime;
1384     return mDisplayPts + delta;
1385 }
1386
1387 /* Called by VideoState::updateVideo to display the next video frame. */
1388 void VideoState::display(SDL_Window *screen, SDL_Renderer *renderer)
1389 {
1390     if(!mImage)
1391         return;
1392
1393     double aspect_ratio;
1394     int win_w, win_h;
1395     int w, h, x, y;
1396
1397     if(mCodecCtx->sample_aspect_ratio.num == 0)
1398         aspect_ratio = 0.0;
1399     else
1400     {
1401         aspect_ratio = av_q2d(mCodecCtx->sample_aspect_ratio) * mCodecCtx->width /
1402                        mCodecCtx->height;
1403     }
1404     if(aspect_ratio <= 0.0)
1405         aspect_ratio = static_cast<double>(mCodecCtx->width) / mCodecCtx->height;
1406
1407     SDL_GetWindowSize(screen, &win_w, &win_h);
1408     h = win_h;
1409     w = (static_cast<int>(std::rint(h * aspect_ratio)) + 3) & ~3;
1410     if(w > win_w)
1411     {
1412         w = win_w;
1413         h = (static_cast<int>(std::rint(w / aspect_ratio)) + 3) & ~3;
1414     }
1415     x = (win_w - w) / 2;
1416     y = (win_h - h) / 2;
1417
1418     SDL_Rect src_rect{ 0, 0, mWidth, mHeight };
1419     SDL_Rect dst_rect{ x, y, w, h };
1420     SDL_RenderCopy(renderer, mImage, &src_rect, &dst_rect);
1421     SDL_RenderPresent(renderer);
1422 }
1423
1424 /* Called regularly on the main thread where the SDL_Renderer was created. It
1425  * handles updating the textures of decoded frames and displaying the latest
1426  * frame.
1427  */
1428 void VideoState::updateVideo(SDL_Window *screen, SDL_Renderer *renderer, bool redraw)
1429 {
1430     size_t read_idx{mPictQRead.load(std::memory_order_relaxed)};
1431     Picture *vp{&mPictQ[read_idx]};
1432
1433     auto clocktime = mMovie.getMasterClock();
1434     bool updated{false};
1435     while(1)
1436     {
1437         size_t next_idx{(read_idx+1)%mPictQ.size()};
1438         if(next_idx == mPictQWrite.load(std::memory_order_acquire))
1439             break;
1440         Picture *nextvp{&mPictQ[next_idx]};
1441         if(clocktime < nextvp->mPts)
1442             break;
1443
1444         vp = nextvp;
1445         updated = true;
1446         read_idx = next_idx;
1447     }
1448     if(mMovie.mQuit.load(std::memory_order_relaxed))
1449     {
1450         if(mEOS)
1451             mFinalUpdate = true;
1452         mPictQRead.store(read_idx, std::memory_order_release);
1453         std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1454         mPictQCond.notify_one();
1455         return;
1456     }
1457
1458     if(updated)
1459     {
1460         mPictQRead.store(read_idx, std::memory_order_release);
1461         std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1462         mPictQCond.notify_one();
1463
1464         /* allocate or resize the buffer! */
1465         bool fmt_updated{false};
1466         if(!mImage || mWidth != mCodecCtx->width || mHeight != mCodecCtx->height)
1467         {
1468             fmt_updated = true;
1469             if(mImage)
1470                 SDL_DestroyTexture(mImage);
1471             mImage = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING,
1472                 mCodecCtx->coded_width, mCodecCtx->coded_height);
1473             if(!mImage)
1474                 std::cerr<< "Failed to create YV12 texture!" <<std::endl;
1475             mWidth = mCodecCtx->width;
1476             mHeight = mCodecCtx->height;
1477
1478             if(mFirstUpdate && mWidth > 0 && mHeight > 0)
1479             {
1480                 /* For the first update, set the window size to the video size. */
1481                 mFirstUpdate = false;
1482
1483                 int w{mWidth};
1484                 int h{mHeight};
1485                 if(mCodecCtx->sample_aspect_ratio.den != 0)
1486                 {
1487                     double aspect_ratio = av_q2d(mCodecCtx->sample_aspect_ratio);
1488                     if(aspect_ratio >= 1.0)
1489                         w = static_cast<int>(w*aspect_ratio + 0.5);
1490                     else if(aspect_ratio > 0.0)
1491                         h = static_cast<int>(h/aspect_ratio + 0.5);
1492                 }
1493                 SDL_SetWindowSize(screen, w, h);
1494             }
1495         }
1496
1497         if(mImage)
1498         {
1499             AVFrame *frame{vp->mFrame.get()};
1500             void *pixels{nullptr};
1501             int pitch{0};
1502
1503             if(mCodecCtx->pix_fmt == AV_PIX_FMT_YUV420P)
1504                 SDL_UpdateYUVTexture(mImage, nullptr,
1505                     frame->data[0], frame->linesize[0],
1506                     frame->data[1], frame->linesize[1],
1507                     frame->data[2], frame->linesize[2]
1508                 );
1509             else if(SDL_LockTexture(mImage, nullptr, &pixels, &pitch) != 0)
1510                 std::cerr<< "Failed to lock texture" <<std::endl;
1511             else
1512             {
1513                 // Convert the image into YUV format that SDL uses
1514                 int coded_w{mCodecCtx->coded_width};
1515                 int coded_h{mCodecCtx->coded_height};
1516                 int w{mCodecCtx->width};
1517                 int h{mCodecCtx->height};
1518                 if(!mSwscaleCtx || fmt_updated)
1519                 {
1520                     mSwscaleCtx.reset(sws_getContext(
1521                         w, h, mCodecCtx->pix_fmt,
1522                         w, h, AV_PIX_FMT_YUV420P, 0,
1523                         nullptr, nullptr, nullptr
1524                     ));
1525                 }
1526
1527                 /* point pict at the queue */
1528                 uint8_t *pict_data[3];
1529                 pict_data[0] = static_cast<uint8_t*>(pixels);
1530                 pict_data[1] = pict_data[0] + coded_w*coded_h;
1531                 pict_data[2] = pict_data[1] + coded_w*coded_h/4;
1532
1533                 int pict_linesize[3];
1534                 pict_linesize[0] = pitch;
1535                 pict_linesize[1] = pitch / 2;
1536                 pict_linesize[2] = pitch / 2;
1537
1538                 sws_scale(mSwscaleCtx.get(), reinterpret_cast<uint8_t**>(frame->data), frame->linesize,
1539                     0, h, pict_data, pict_linesize);
1540                 SDL_UnlockTexture(mImage);
1541             }
1542         }
1543
1544         redraw = true;
1545     }
1546
1547     if(redraw)
1548     {
1549         /* Show the picture! */
1550         display(screen, renderer);
1551     }
1552
1553     if(updated)
1554     {
1555         auto disp_time = get_avtime();
1556
1557         std::lock_guard<std::mutex> _{mDispPtsMutex};
1558         mDisplayPts = vp->mPts;
1559         mDisplayPtsTime = disp_time;
1560     }
1561     if(mEOS.load(std::memory_order_acquire))
1562     {
1563         if((read_idx+1)%mPictQ.size() == mPictQWrite.load(std::memory_order_acquire))
1564         {
1565             mFinalUpdate = true;
1566             std::unique_lock<std::mutex>{mPictQMutex}.unlock();
1567             mPictQCond.notify_one();
1568         }
1569     }
1570 }
1571
1572 int VideoState::handler()
1573 {
1574     std::for_each(mPictQ.begin(), mPictQ.end(),
1575         [](Picture &pict) -> void
1576         { pict.mFrame = AVFramePtr{av_frame_alloc()}; });
1577
1578     /* Prefill the codec buffer. */
1579     do {
1580         const int ret{mPackets.sendTo(mCodecCtx.get())};
1581         if(ret == AVERROR(EAGAIN) || ret == AVErrorEOF)
1582             break;
1583     } while(1);
1584
1585     {
1586         std::lock_guard<std::mutex> _{mDispPtsMutex};
1587         mDisplayPtsTime = get_avtime();
1588     }
1589
1590     auto current_pts = nanoseconds::zero();
1591     while(!mMovie.mQuit.load(std::memory_order_relaxed))
1592     {
1593         size_t write_idx{mPictQWrite.load(std::memory_order_relaxed)};
1594         Picture *vp{&mPictQ[write_idx]};
1595
1596         /* Retrieve video frame. */
1597         AVFrame *decoded_frame{vp->mFrame.get()};
1598         int ret;
1599         while((ret=avcodec_receive_frame(mCodecCtx.get(), decoded_frame)) == AVERROR(EAGAIN))
1600             mPackets.sendTo(mCodecCtx.get());
1601         if(ret != 0)
1602         {
1603             if(ret == AVErrorEOF) break;
1604             std::cerr<< "Failed to receive frame: "<<ret <<std::endl;
1605             continue;
1606         }
1607
1608         /* Get the PTS for this frame. */
1609         if(decoded_frame->best_effort_timestamp != AVNoPtsValue)
1610             current_pts = duration_cast<nanoseconds>(seconds_d64{av_q2d(mStream->time_base) *
1611                 static_cast<double>(decoded_frame->best_effort_timestamp)});
1612         vp->mPts = current_pts;
1613
1614         /* Update the video clock to the next expected PTS. */
1615         auto frame_delay = av_q2d(mCodecCtx->time_base);
1616         frame_delay += decoded_frame->repeat_pict * (frame_delay * 0.5);
1617         current_pts += duration_cast<nanoseconds>(seconds_d64{frame_delay});
1618
1619         /* Put the frame in the queue to be loaded into a texture and displayed
1620          * by the rendering thread.
1621          */
1622         write_idx = (write_idx+1)%mPictQ.size();
1623         mPictQWrite.store(write_idx, std::memory_order_release);
1624
1625         /* Send a packet now so it's hopefully ready by the time it's needed. */
1626         mPackets.sendTo(mCodecCtx.get());
1627
1628         if(write_idx == mPictQRead.load(std::memory_order_acquire))
1629         {
1630             /* Wait until we have space for a new pic */
1631             std::unique_lock<std::mutex> lock{mPictQMutex};
1632             while(write_idx == mPictQRead.load(std::memory_order_acquire) &&
1633                 !mMovie.mQuit.load(std::memory_order_relaxed))
1634                 mPictQCond.wait(lock);
1635         }
1636     }
1637     mEOS = true;
1638
1639     std::unique_lock<std::mutex> lock{mPictQMutex};
1640     while(!mFinalUpdate) mPictQCond.wait(lock);
1641
1642     return 0;
1643 }
1644
1645
1646 int MovieState::decode_interrupt_cb(void *ctx)
1647 {
1648     return static_cast<MovieState*>(ctx)->mQuit.load(std::memory_order_relaxed);
1649 }
1650
1651 bool MovieState::prepare()
1652 {
1653     AVIOContext *avioctx{nullptr};
1654     AVIOInterruptCB intcb{decode_interrupt_cb, this};
1655     if(avio_open2(&avioctx, mFilename.c_str(), AVIO_FLAG_READ, &intcb, nullptr))
1656     {
1657         std::cerr<< "Failed to open "<<mFilename <<std::endl;
1658         return false;
1659     }
1660     mIOContext.reset(avioctx);
1661
1662     /* Open movie file. If avformat_open_input fails it will automatically free
1663      * this context, so don't set it onto a smart pointer yet.
1664      */
1665     AVFormatContext *fmtctx{avformat_alloc_context()};
1666     fmtctx->pb = mIOContext.get();
1667     fmtctx->interrupt_callback = intcb;
1668     if(avformat_open_input(&fmtctx, mFilename.c_str(), nullptr, nullptr) != 0)
1669     {
1670         std::cerr<< "Failed to open "<<mFilename <<std::endl;
1671         return false;
1672     }
1673     mFormatCtx.reset(fmtctx);
1674
1675     /* Retrieve stream information */
1676     if(avformat_find_stream_info(mFormatCtx.get(), nullptr) < 0)
1677     {
1678         std::cerr<< mFilename<<": failed to find stream info" <<std::endl;
1679         return false;
1680     }
1681
1682     /* Dump information about file onto standard error */
1683     av_dump_format(mFormatCtx.get(), 0, mFilename.c_str(), 0);
1684
1685     mParseThread = std::thread{std::mem_fn(&MovieState::parse_handler), this};
1686     return true;
1687 }
1688
1689 void MovieState::setTitle(SDL_Window *window)
1690 {
1691     auto pos1 = mFilename.rfind('/');
1692     auto pos2 = mFilename.rfind('\\');
1693     auto fpos = ((pos1 == std::string::npos) ? pos2 :
1694                  (pos2 == std::string::npos) ? pos1 :
1695                  std::max(pos1, pos2)) + 1;
1696     SDL_SetWindowTitle(window, (mFilename.substr(fpos)+" - "+AppName).c_str());
1697 }
1698
1699 nanoseconds MovieState::getClock()
1700 {
1701     if(mClockBase == microseconds::min())
1702         return nanoseconds::zero();
1703     return get_avtime() - mClockBase;
1704 }
1705
1706 nanoseconds MovieState::getMasterClock()
1707 {
1708     if(mAVSyncType == SyncMaster::Video)
1709         return mVideo.getClock();
1710     if(mAVSyncType == SyncMaster::Audio)
1711         return mAudio.getClock();
1712     return getClock();
1713 }
1714
1715 nanoseconds MovieState::getDuration()
1716 { return std::chrono::duration<int64_t,std::ratio<1,AV_TIME_BASE>>(mFormatCtx->duration); }
1717
1718 int MovieState::streamComponentOpen(unsigned int stream_index)
1719 {
1720     if(stream_index >= mFormatCtx->nb_streams)
1721         return -1;
1722
1723     /* Get a pointer to the codec context for the stream, and open the
1724      * associated codec.
1725      */
1726     AVCodecCtxPtr avctx{avcodec_alloc_context3(nullptr)};
1727     if(!avctx) return -1;
1728
1729     if(avcodec_parameters_to_context(avctx.get(), mFormatCtx->streams[stream_index]->codecpar))
1730         return -1;
1731
1732     AVCodec *codec{avcodec_find_decoder(avctx->codec_id)};
1733     if(!codec || avcodec_open2(avctx.get(), codec, nullptr) < 0)
1734     {
1735         std::cerr<< "Unsupported codec: "<<avcodec_get_name(avctx->codec_id)
1736             << " (0x"<<std::hex<<avctx->codec_id<<std::dec<<")" <<std::endl;
1737         return -1;
1738     }
1739
1740     /* Initialize and start the media type handler */
1741     switch(avctx->codec_type)
1742     {
1743         case AVMEDIA_TYPE_AUDIO:
1744             mAudio.mStream = mFormatCtx->streams[stream_index];
1745             mAudio.mCodecCtx = std::move(avctx);
1746             break;
1747
1748         case AVMEDIA_TYPE_VIDEO:
1749             mVideo.mStream = mFormatCtx->streams[stream_index];
1750             mVideo.mCodecCtx = std::move(avctx);
1751             break;
1752
1753         default:
1754             return -1;
1755     }
1756
1757     return static_cast<int>(stream_index);
1758 }
1759
1760 int MovieState::parse_handler()
1761 {
1762     auto &audio_queue = mAudio.mPackets;
1763     auto &video_queue = mVideo.mPackets;
1764
1765     int video_index{-1};
1766     int audio_index{-1};
1767
1768     /* Find the first video and audio streams */
1769     for(unsigned int i{0u};i < mFormatCtx->nb_streams;i++)
1770     {
1771         auto codecpar = mFormatCtx->streams[i]->codecpar;
1772         if(codecpar->codec_type == AVMEDIA_TYPE_VIDEO && !DisableVideo && video_index < 0)
1773             video_index = streamComponentOpen(i);
1774         else if(codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_index < 0)
1775             audio_index = streamComponentOpen(i);
1776     }
1777
1778     if(video_index < 0 && audio_index < 0)
1779     {
1780         std::cerr<< mFilename<<": could not open codecs" <<std::endl;
1781         mQuit = true;
1782     }
1783
1784     /* Set the base time 750ms ahead of the current av time. */
1785     mClockBase = get_avtime() + milliseconds{750};
1786
1787     if(audio_index >= 0)
1788         mAudioThread = std::thread{std::mem_fn(&AudioState::handler), &mAudio};
1789     if(video_index >= 0)
1790         mVideoThread = std::thread{std::mem_fn(&VideoState::handler), &mVideo};
1791
1792     /* Main packet reading/dispatching loop */
1793     while(!mQuit.load(std::memory_order_relaxed))
1794     {
1795         AVPacket packet;
1796         if(av_read_frame(mFormatCtx.get(), &packet) < 0)
1797             break;
1798
1799         /* Copy the packet into the queue it's meant for. */
1800         if(packet.stream_index == video_index)
1801         {
1802             while(!mQuit.load(std::memory_order_acquire) && !video_queue.put(&packet))
1803                 std::this_thread::sleep_for(milliseconds{100});
1804         }
1805         else if(packet.stream_index == audio_index)
1806         {
1807             while(!mQuit.load(std::memory_order_acquire) && !audio_queue.put(&packet))
1808                 std::this_thread::sleep_for(milliseconds{100});
1809         }
1810
1811         av_packet_unref(&packet);
1812     }
1813     /* Finish the queues so the receivers know nothing more is coming. */
1814     if(mVideo.mCodecCtx) video_queue.setFinished();
1815     if(mAudio.mCodecCtx) audio_queue.setFinished();
1816
1817     /* all done - wait for it */
1818     if(mVideoThread.joinable())
1819         mVideoThread.join();
1820     if(mAudioThread.joinable())
1821         mAudioThread.join();
1822
1823     mVideo.mEOS = true;
1824     std::unique_lock<std::mutex> lock{mVideo.mPictQMutex};
1825     while(!mVideo.mFinalUpdate)
1826         mVideo.mPictQCond.wait(lock);
1827     lock.unlock();
1828
1829     SDL_Event evt{};
1830     evt.user.type = FF_MOVIE_DONE_EVENT;
1831     SDL_PushEvent(&evt);
1832
1833     return 0;
1834 }
1835
1836
1837 // Helper class+method to print the time with human-readable formatting.
1838 struct PrettyTime {
1839     seconds mTime;
1840 };
1841 std::ostream &operator<<(std::ostream &os, const PrettyTime &rhs)
1842 {
1843     using hours = std::chrono::hours;
1844     using minutes = std::chrono::minutes;
1845
1846     seconds t{rhs.mTime};
1847     if(t.count() < 0)
1848     {
1849         os << '-';
1850         t *= -1;
1851     }
1852
1853     // Only handle up to hour formatting
1854     if(t >= hours{1})
1855         os << duration_cast<hours>(t).count() << 'h' << std::setfill('0') << std::setw(2)
1856            << (duration_cast<minutes>(t).count() % 60) << 'm';
1857     else
1858         os << duration_cast<minutes>(t).count() << 'm' << std::setfill('0');
1859     os << std::setw(2) << (duration_cast<seconds>(t).count() % 60) << 's' << std::setw(0)
1860        << std::setfill(' ');
1861     return os;
1862 }
1863
1864 } // namespace
1865
1866
1867 int main(int argc, char *argv[])
1868 {
1869     std::unique_ptr<MovieState> movState;
1870
1871     if(argc < 2)
1872     {
1873         std::cerr<< "Usage: "<<argv[0]<<" [-device <device name>] [-direct] <files...>" <<std::endl;
1874         return 1;
1875     }
1876     /* Register all formats and codecs */
1877 #if !(LIBAVFORMAT_VERSION_INT >= AV_VERSION_INT(58, 9, 100))
1878     av_register_all();
1879 #endif
1880     /* Initialize networking protocols */
1881     avformat_network_init();
1882
1883     if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS))
1884     {
1885         std::cerr<< "Could not initialize SDL - <<"<<SDL_GetError() <<std::endl;
1886         return 1;
1887     }
1888
1889     /* Make a window to put our video */
1890     SDL_Window *screen{SDL_CreateWindow(AppName.c_str(), 0, 0, 640, 480, SDL_WINDOW_RESIZABLE)};
1891     if(!screen)
1892     {
1893         std::cerr<< "SDL: could not set video mode - exiting" <<std::endl;
1894         return 1;
1895     }
1896     /* Make a renderer to handle the texture image surface and rendering. */
1897     Uint32 render_flags{SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC};
1898     SDL_Renderer *renderer{SDL_CreateRenderer(screen, -1, render_flags)};
1899     if(renderer)
1900     {
1901         SDL_RendererInfo rinf{};
1902         bool ok{false};
1903
1904         /* Make sure the renderer supports IYUV textures. If not, fallback to a
1905          * software renderer. */
1906         if(SDL_GetRendererInfo(renderer, &rinf) == 0)
1907         {
1908             for(Uint32 i{0u};!ok && i < rinf.num_texture_formats;i++)
1909                 ok = (rinf.texture_formats[i] == SDL_PIXELFORMAT_IYUV);
1910         }
1911         if(!ok)
1912         {
1913             std::cerr<< "IYUV pixelformat textures not supported on renderer "<<rinf.name <<std::endl;
1914             SDL_DestroyRenderer(renderer);
1915             renderer = nullptr;
1916         }
1917     }
1918     if(!renderer)
1919     {
1920         render_flags = SDL_RENDERER_SOFTWARE | SDL_RENDERER_PRESENTVSYNC;
1921         renderer = SDL_CreateRenderer(screen, -1, render_flags);
1922     }
1923     if(!renderer)
1924     {
1925         std::cerr<< "SDL: could not create renderer - exiting" <<std::endl;
1926         return 1;
1927     }
1928     SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
1929     SDL_RenderFillRect(renderer, nullptr);
1930     SDL_RenderPresent(renderer);
1931
1932     /* Open an audio device */
1933     ++argv; --argc;
1934     if(InitAL(&argv, &argc))
1935     {
1936         std::cerr<< "Failed to set up audio device" <<std::endl;
1937         return 1;
1938     }
1939
1940     {
1941         auto device = alcGetContextsDevice(alcGetCurrentContext());
1942         if(alcIsExtensionPresent(device, "ALC_SOFT_device_clock"))
1943         {
1944             std::cout<< "Found ALC_SOFT_device_clock" <<std::endl;
1945             alcGetInteger64vSOFT = reinterpret_cast<LPALCGETINTEGER64VSOFT>(
1946                 alcGetProcAddress(device, "alcGetInteger64vSOFT")
1947             );
1948         }
1949     }
1950
1951     if(alIsExtensionPresent("AL_SOFT_source_latency"))
1952     {
1953         std::cout<< "Found AL_SOFT_source_latency" <<std::endl;
1954         alGetSourcei64vSOFT = reinterpret_cast<LPALGETSOURCEI64VSOFT>(
1955             alGetProcAddress("alGetSourcei64vSOFT")
1956         );
1957     }
1958 #ifdef AL_SOFT_events
1959     if(alIsExtensionPresent("AL_SOFTX_events"))
1960     {
1961         std::cout<< "Found AL_SOFT_events" <<std::endl;
1962         alEventControlSOFT = reinterpret_cast<LPALEVENTCONTROLSOFT>(
1963             alGetProcAddress("alEventControlSOFT"));
1964         alEventCallbackSOFT = reinterpret_cast<LPALEVENTCALLBACKSOFT>(
1965             alGetProcAddress("alEventCallbackSOFT"));
1966     }
1967 #endif
1968 #ifdef AL_SOFT_callback_buffer
1969     if(alIsExtensionPresent("AL_SOFTX_callback_buffer"))
1970     {
1971         std::cout<< "Found AL_SOFT_callback_buffer" <<std::endl;
1972         alBufferCallbackSOFT = reinterpret_cast<LPALBUFFERCALLBACKSOFT>(
1973             alGetProcAddress("alBufferCallbackSOFT"));
1974     }
1975 #endif
1976
1977     int fileidx{0};
1978     for(;fileidx < argc;++fileidx)
1979     {
1980         if(strcmp(argv[fileidx], "-direct") == 0)
1981         {
1982             if(alIsExtensionPresent("AL_SOFT_direct_channels_remix"))
1983             {
1984                 std::cout<< "Found AL_SOFT_direct_channels_remix" <<std::endl;
1985                 DirectOutMode = AL_REMIX_UNMATCHED_SOFT;
1986             }
1987             else if(alIsExtensionPresent("AL_SOFT_direct_channels"))
1988             {
1989                 std::cout<< "Found AL_SOFT_direct_channels" <<std::endl;
1990                 DirectOutMode = AL_DROP_UNMATCHED_SOFT;
1991             }
1992             else
1993                 std::cerr<< "AL_SOFT_direct_channels not supported for direct output" <<std::endl;
1994         }
1995         else if(strcmp(argv[fileidx], "-wide") == 0)
1996         {
1997             if(!alIsExtensionPresent("AL_EXT_STEREO_ANGLES"))
1998                 std::cerr<< "AL_EXT_STEREO_ANGLES not supported for wide stereo" <<std::endl;
1999             else
2000             {
2001                 std::cout<< "Found AL_EXT_STEREO_ANGLES" <<std::endl;
2002                 EnableWideStereo = true;
2003             }
2004         }
2005         else if(strcmp(argv[fileidx], "-novideo") == 0)
2006             DisableVideo = true;
2007         else
2008             break;
2009     }
2010
2011     while(fileidx < argc && !movState)
2012     {
2013         movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
2014         if(!movState->prepare()) movState = nullptr;
2015     }
2016     if(!movState)
2017     {
2018         std::cerr<< "Could not start a video" <<std::endl;
2019         return 1;
2020     }
2021     movState->setTitle(screen);
2022
2023     /* Default to going to the next movie at the end of one. */
2024     enum class EomAction {
2025         Next, Quit
2026     } eom_action{EomAction::Next};
2027     seconds last_time{seconds::min()};
2028     while(1)
2029     {
2030         SDL_Event event{};
2031         int have_evt{SDL_WaitEventTimeout(&event, 10)};
2032
2033         auto cur_time = std::chrono::duration_cast<seconds>(movState->getMasterClock());
2034         if(cur_time != last_time)
2035         {
2036             auto end_time = std::chrono::duration_cast<seconds>(movState->getDuration());
2037             std::cout<< "    \r "<<PrettyTime{cur_time}<<" / "<<PrettyTime{end_time} <<std::flush;
2038             last_time = cur_time;
2039         }
2040
2041         bool force_redraw{false};
2042         if(have_evt) do {
2043             switch(event.type)
2044             {
2045             case SDL_KEYDOWN:
2046                 switch(event.key.keysym.sym)
2047                 {
2048                 case SDLK_ESCAPE:
2049                     movState->mQuit = true;
2050                     eom_action = EomAction::Quit;
2051                     break;
2052
2053                 case SDLK_n:
2054                     movState->mQuit = true;
2055                     eom_action = EomAction::Next;
2056                     break;
2057
2058                 default:
2059                     break;
2060                 }
2061                 break;
2062
2063             case SDL_WINDOWEVENT:
2064                 switch(event.window.event)
2065                 {
2066                 case SDL_WINDOWEVENT_RESIZED:
2067                     SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
2068                     SDL_RenderFillRect(renderer, nullptr);
2069                     force_redraw = true;
2070                     break;
2071
2072                 case SDL_WINDOWEVENT_EXPOSED:
2073                     force_redraw = true;
2074                     break;
2075
2076                 default:
2077                     break;
2078                 }
2079                 break;
2080
2081             case SDL_QUIT:
2082                 movState->mQuit = true;
2083                 eom_action = EomAction::Quit;
2084                 break;
2085
2086             case FF_MOVIE_DONE_EVENT:
2087                 std::cout<<'\n';
2088                 last_time = seconds::min();
2089                 if(eom_action != EomAction::Quit)
2090                 {
2091                     movState = nullptr;
2092                     while(fileidx < argc && !movState)
2093                     {
2094                         movState = std::unique_ptr<MovieState>{new MovieState{argv[fileidx++]}};
2095                         if(!movState->prepare()) movState = nullptr;
2096                     }
2097                     if(movState)
2098                     {
2099                         movState->setTitle(screen);
2100                         break;
2101                     }
2102                 }
2103
2104                 /* Nothing more to play. Shut everything down and quit. */
2105                 movState = nullptr;
2106
2107                 CloseAL();
2108
2109                 SDL_DestroyRenderer(renderer);
2110                 renderer = nullptr;
2111                 SDL_DestroyWindow(screen);
2112                 screen = nullptr;
2113
2114                 SDL_Quit();
2115                 exit(0);
2116
2117             default:
2118                 break;
2119             }
2120         } while(SDL_PollEvent(&event));
2121
2122         movState->mVideo.updateVideo(screen, renderer, force_redraw);
2123     }
2124
2125     std::cerr<< "SDL_WaitEvent error - "<<SDL_GetError() <<std::endl;
2126     return 1;
2127 }