xbmc/cores/VideoPlayer/VideoPlayerVideo.cpp

   1 /*
   2  *  Copyright (C) 2005-2018 Team Kodi
   3  *  This file is part of Kodi - https://kodi.tv
   4  *
   5  *  SPDX-License-Identifier: GPL-2.0-or-later
   6  *  See LICENSES/README.md for more information.
   7  */
   8
   9 #include "VideoPlayerVideo.h"
  10
  11 #include "DVDCodecs/DVDCodecUtils.h"
  12 #include "DVDCodecs/DVDFactoryCodec.h"
  13 #include "DVDCodecs/Overlay/DVDOverlay.h"
  14 #include "DVDCodecs/Video/DVDVideoCodecFFmpeg.h"
  15 #include "ServiceBroker.h"
  16 #include "cores/VideoPlayer/Interface/DemuxPacket.h"
  17 #include "cores/VideoPlayer/Interface/TimingConstants.h"
  18 #include "settings/AdvancedSettings.h"
  19 #include "settings/Settings.h"
  20 #include "settings/SettingsComponent.h"
  21 #include "utils/MathUtils.h"
  22 #include "utils/log.h"
  23 #include "windowing/GraphicContext.h"
  24 #include "windowing/WinSystem.h"
  25
  26 #include <iomanip>
  27 #include <iterator>
  28 #include <memory>
  29 #include <mutex>
  30 #include <numeric>
  31 #include <sstream>
  32
  33 using namespace std::chrono_literals;
  34
  35 class CDVDMsgVideoCodecChange : public CDVDMsg
  36 {
  37 public:
  38   CDVDMsgVideoCodecChange(const CDVDStreamInfo& hints, std::unique_ptr<CDVDVideoCodec> codec)
  39     : CDVDMsg(GENERAL_STREAMCHANGE), m_codec(std::move(codec)), m_hints(hints)
  40   {}
  41   ~CDVDMsgVideoCodecChange() override = default;
  42
  43   std::unique_ptr<CDVDVideoCodec> m_codec;
  44   CDVDStreamInfo  m_hints;
  45 };
  46
  47 CVideoPlayerVideo::CVideoPlayerVideo(CDVDClock* pClock,
  48                                      CDVDOverlayContainer* pOverlayContainer,
  49                                      CDVDMessageQueue& parent,
  50                                      CRenderManager& renderManager,
  51                                      CProcessInfo& processInfo,
  52                                      double messageQueueTimeSize)
  53   : CThread("VideoPlayerVideo"),
  54     IDVDStreamPlayerVideo(processInfo),
  55     m_messageQueue("video"),
  56     m_messageParent(parent),
  57     m_renderManager(renderManager)
  58 {
  59   m_pClock = pClock;
  60   m_pOverlayContainer = pOverlayContainer;
  61   m_speed = DVD_PLAYSPEED_NORMAL;
  62
  63   m_bRenderSubs = false;
  64   m_paused = false;
  65   m_syncState = IDVDStreamPlayer::SYNC_STARTING;
  66   m_iSubtitleDelay = 0;
  67   m_iLateFrames = 0;
  68   m_iDroppedRequest = 0;
  69   m_fForcedAspectRatio = 0;
  70
  71   const int sizeMB = CServiceBroker::GetSettingsComponent()->GetSettings()->GetInt(
  72       CSettings::SETTING_VIDEOPLAYER_QUEUEDATASIZE);
  73
  74   m_messageQueue.SetMaxDataSize(sizeMB * 1024 * 1024);
  75   m_messageQueue.SetMaxTimeSize(messageQueueTimeSize);
  76
  77   m_iDroppedFrames = 0;
  78   m_fFrameRate = 25;
  79   m_fStableFrameRate = 0.0;
  80   m_iFrameRateCount = 0;
  81   m_bAllowDrop = false;
  82   m_iFrameRateErr = 0;
  83   m_iFrameRateLength = 0;
  84   m_bFpsInvalid = false;
  85 }
  86
  87 CVideoPlayerVideo::~CVideoPlayerVideo()
  88 {
  89   m_bAbortOutput = true;
  90   StopThread();
  91 }
  92
  93 double CVideoPlayerVideo::GetOutputDelay()
  94 {
  95   double time = m_messageQueue.GetPacketCount(CDVDMsg::DEMUXER_PACKET);
  96   if( m_fFrameRate )
  97     time = (time * DVD_TIME_BASE) / m_fFrameRate;
  98   else
  99     time = 0.0;
 100
 101   if( m_speed != 0 )
 102     time = time * DVD_PLAYSPEED_NORMAL / abs(m_speed);
 103
 104   return time;
 105 }
 106
 107 bool CVideoPlayerVideo::OpenStream(CDVDStreamInfo hint)
 108 {
 109   if (hint.flags & AV_DISPOSITION_ATTACHED_PIC)
 110     return false;
 111   if (!hint.extradata)
 112   {
 113     // codecs which require extradata
 114     // clang-format off
 115     if (hint.codec == AV_CODEC_ID_NONE ||
 116         hint.codec == AV_CODEC_ID_MPEG1VIDEO ||
 117         hint.codec == AV_CODEC_ID_MPEG2VIDEO ||
 118         (hint.codec == AV_CODEC_ID_H264 && (hint.codec_tag == 0 || hint.codec_tag == MKTAG('a','v','c','1') || hint.codec_tag == MKTAG('a','v','c','2'))) ||
 119         hint.codec == AV_CODEC_ID_HEVC ||
 120         hint.codec == AV_CODEC_ID_MPEG4 ||
 121         hint.codec == AV_CODEC_ID_WMV3 ||
 122         hint.codec == AV_CODEC_ID_VC1 ||
 123         hint.codec == AV_CODEC_ID_AV1)
 124     {
 125       CLog::LogF(LOGERROR, "Codec id {} require extradata.", hint.codec);
 126       return false;
 127     }
 128     // clang-format on
 129   }
 130
 131   CLog::Log(LOGINFO, "Creating video codec with codec id: {}", hint.codec);
 132
 133   if (m_messageQueue.IsInited())
 134   {
 135     if (m_pVideoCodec && !m_processInfo.IsVideoHwDecoder())
 136     {
 137       hint.codecOptions |= CODEC_ALLOW_FALLBACK;
 138     }
 139
 140     std::unique_ptr<CDVDVideoCodec> codec = CDVDFactoryCodec::CreateVideoCodec(hint, m_processInfo);
 141     if (!codec)
 142     {
 143       CLog::Log(LOGINFO, "CVideoPlayerVideo::OpenStream - could not open video codec");
 144     }
 145
 146     SendMessage(std::make_shared<CDVDMsgVideoCodecChange>(hint, std::move(codec)), 0);
 147   }
 148   else
 149   {
 150     m_processInfo.ResetVideoCodecInfo();
 151     hint.codecOptions |= CODEC_ALLOW_FALLBACK;
 152
 153     std::unique_ptr<CDVDVideoCodec> codec = CDVDFactoryCodec::CreateVideoCodec(hint, m_processInfo);
 154     if (!codec)
 155     {
 156       CLog::Log(LOGERROR, "CVideoPlayerVideo::OpenStream - could not open video codec");
 157       return false;
 158     }
 159
 160     OpenStream(hint, std::move(codec));
 161     CLog::Log(LOGINFO, "Creating video thread");
 162     m_messageQueue.Init();
 163     m_processInfo.SetLevelVQ(0);
 164     Create();
 165   }
 166   return true;
 167 }
 168
 169 void CVideoPlayerVideo::OpenStream(CDVDStreamInfo& hint, std::unique_ptr<CDVDVideoCodec> codec)
 170 {
 171   CLog::Log(LOGDEBUG, "CVideoPlayerVideo::OpenStream - open stream with codec id: {}", hint.codec);
 172
 173   m_processInfo.GetVideoBufferManager().ReleasePools();
 174
 175   //reported fps is usually not completely correct
 176   if (hint.fpsrate && hint.fpsscale)
 177   {
 178     m_fFrameRate = DVD_TIME_BASE / CDVDCodecUtils::NormalizeFrameduration(
 179                                        (double)DVD_TIME_BASE * hint.fpsscale / hint.fpsrate);
 180
 181     m_bFpsInvalid = false;
 182     m_processInfo.SetVideoFps(static_cast<float>(m_fFrameRate));
 183   }
 184   else
 185   {
 186     m_fFrameRate = 25;
 187     m_bFpsInvalid = true;
 188     m_processInfo.SetVideoFps(0);
 189   }
 190
 191   m_ptsTracker.ResetVFRDetection();
 192   ResetFrameRateCalc();
 193
 194   m_iDroppedRequest = 0;
 195   m_iLateFrames = 0;
 196
 197   if( m_fFrameRate > 120 || m_fFrameRate < 5 )
 198   {
 199     CLog::Log(LOGERROR,
 200               "CVideoPlayerVideo::OpenStream - Invalid framerate {}, using forced 25fps and just "
 201               "trust timestamps",
 202               (int)m_fFrameRate);
 203     m_fFrameRate = 25;
 204   }
 205
 206   // use aspect in stream if available
 207   if (hint.forced_aspect)
 208     m_fForcedAspectRatio = static_cast<float>(hint.aspect);
 209   else
 210     m_fForcedAspectRatio = 0.0f;
 211
 212   if (m_pVideoCodec && m_pVideoCodec->Reconfigure(hint))
 213   {
 214     // reuse old decoder
 215     codec = std::move(m_pVideoCodec);
 216   }
 217
 218   m_pVideoCodec.reset();
 219
 220   if (!codec)
 221   {
 222     CLog::Log(LOGINFO, "Creating video codec with codec id: {}", hint.codec);
 223     hint.codecOptions |= CODEC_ALLOW_FALLBACK;
 224     codec = CDVDFactoryCodec::CreateVideoCodec(hint, m_processInfo);
 225     if (!codec)
 226     {
 227       CLog::Log(LOGERROR, "CVideoPlayerVideo::OpenStream - could not open video codec");
 228       m_messageParent.Put(std::make_shared<CDVDMsg>(CDVDMsg::PLAYER_ABORT));
 229       StopThread();
 230     }
 231   }
 232
 233   m_pVideoCodec = std::move(codec);
 234   m_hints = hint;
 235   m_stalled = m_messageQueue.GetPacketCount(CDVDMsg::DEMUXER_PACKET) == 0;
 236   m_rewindStalled = false;
 237   m_packets.clear();
 238   m_syncState = IDVDStreamPlayer::SYNC_STARTING;
 239   m_renderManager.ShowVideo(false);
 240 }
 241
 242 void CVideoPlayerVideo::CloseStream(bool bWaitForBuffers)
 243 {
 244   // wait until buffers are empty
 245   if (bWaitForBuffers && m_speed > 0)
 246   {
 247     SendMessage(std::make_shared<CDVDMsg>(CDVDMsg::VIDEO_DRAIN), 0);
 248     m_messageQueue.WaitUntilEmpty();
 249   }
 250
 251   m_messageQueue.Abort();
 252
 253   // wait for decode_video thread to end
 254   CLog::Log(LOGINFO, "waiting for video thread to exit");
 255
 256   m_bAbortOutput = true;
 257   StopThread();
 258
 259   m_messageQueue.End();
 260
 261   CLog::Log(LOGINFO, "deleting video codec");
 262   m_pVideoCodec.reset();
 263
 264   if (m_picture.videoBuffer)
 265   {
 266     m_picture.videoBuffer->Release();
 267     m_picture.videoBuffer = nullptr;
 268   }
 269 }
 270
 271 bool CVideoPlayerVideo::AcceptsData() const
 272 {
 273   bool full = m_messageQueue.IsFull();
 274   return !full;
 275 }
 276
 277 bool CVideoPlayerVideo::HasData() const
 278 {
 279   return m_messageQueue.GetDataSize() > 0;
 280 }
 281
 282 bool CVideoPlayerVideo::IsInited() const
 283 {
 284   return m_messageQueue.IsInited();
 285 }
 286
 287 inline void CVideoPlayerVideo::SendMessage(std::shared_ptr<CDVDMsg> pMsg, int priority)
 288 {
 289   m_messageQueue.Put(pMsg, priority);
 290   m_processInfo.SetLevelVQ(m_messageQueue.GetLevel());
 291 }
 292
 293 inline void CVideoPlayerVideo::SendMessageBack(const std::shared_ptr<CDVDMsg>& pMsg, int priority)
 294 {
 295   m_messageQueue.PutBack(pMsg, priority);
 296   m_processInfo.SetLevelVQ(m_messageQueue.GetLevel());
 297 }
 298
 299 inline void CVideoPlayerVideo::FlushMessages()
 300 {
 301   m_messageQueue.Flush();
 302   m_processInfo.SetLevelVQ(m_messageQueue.GetLevel());
 303 }
 304
 305 inline MsgQueueReturnCode CVideoPlayerVideo::GetMessage(std::shared_ptr<CDVDMsg>& pMsg,
 306                                                         std::chrono::milliseconds timeout,
 307                                                         int& priority)
 308 {
 309   MsgQueueReturnCode ret = m_messageQueue.Get(pMsg, timeout, priority);
 310   m_processInfo.SetLevelVQ(m_messageQueue.GetLevel());
 311   return ret;
 312 }
 313
 314 void CVideoPlayerVideo::Process()
 315 {
 316   CLog::Log(LOGINFO, "running thread: video_thread");
 317
 318   double pts = 0;
 319   double frametime = (double)DVD_TIME_BASE / m_fFrameRate;
 320
 321   bool bRequestDrop = false;
 322   int iDropDirective;
 323   bool onlyPrioMsgs = false;
 324
 325   m_videoStats.Start();
 326   m_droppingStats.Reset();
 327   m_iDroppedFrames = 0;
 328   m_rewindStalled = false;
 329   m_outputSate = OUTPUT_NORMAL;
 330
 331   while (!m_bStop)
 332   {
 333     auto timeout = std::chrono::duration_cast<std::chrono::milliseconds>(
 334         std::chrono::duration<double, std::micro>(m_stalled ? frametime : frametime * 10));
 335     int iPriority = 0;
 336
 337     if (m_syncState == IDVDStreamPlayer::SYNC_WAITSYNC)
 338       iPriority = 1;
 339
 340     if (m_paused)
 341       iPriority = 1;
 342
 343     if (onlyPrioMsgs)
 344     {
 345       iPriority = 1;
 346       timeout = 1ms;
 347     }
 348
 349     std::shared_ptr<CDVDMsg> pMsg;
 350     MsgQueueReturnCode ret = GetMessage(pMsg, timeout, iPriority);
 351
 352     onlyPrioMsgs = false;
 353
 354     if (MSGQ_IS_ERROR(ret))
 355     {
 356       if (!m_messageQueue.ReceivedAbortRequest())
 357         CLog::Log(LOGERROR, "MSGQ_IS_ERROR returned true ({})", ret);
 358
 359       break;
 360     }
 361     else if (ret == MSGQ_TIMEOUT)
 362     {
 363       if (m_outputSate == OUTPUT_AGAIN &&
 364           m_picture.videoBuffer)
 365       {
 366         m_outputSate = OutputPicture(&m_picture);
 367         if (m_outputSate == OUTPUT_AGAIN)
 368         {
 369           onlyPrioMsgs = true;
 370           continue;
 371         }
 372       }
 373       // don't ask for a new frame if we can't deliver it to renderer
 374       else if ((m_speed != DVD_PLAYSPEED_PAUSE ||
 375                 m_processInfo.IsFrameAdvance() ||
 376                 m_syncState != IDVDStreamPlayer::SYNC_INSYNC) && !m_paused)
 377       {
 378         if (ProcessDecoderOutput(frametime, pts))
 379         {
 380           onlyPrioMsgs = true;
 381           continue;
 382         }
 383       }
 384
 385       // if we only wanted priority messages, this isn't a stall
 386       if (iPriority)
 387         continue;
 388
 389       //Okey, start rendering at stream fps now instead, we are likely in a stillframe
 390       if (!m_stalled)
 391       {
 392         // squeeze pictures out
 393         while (!m_bStop && m_pVideoCodec)
 394         {
 395           m_pVideoCodec->SetCodecControl(DVD_CODEC_CTRL_DRAIN);
 396           if (!ProcessDecoderOutput(frametime, pts))
 397             break;
 398         }
 399
 400         CLog::Log(LOGDEBUG, "CVideoPlayerVideo - Stillframe detected, switching to forced {:f} fps",
 401                   m_fFrameRate);
 402         m_stalled = true;
 403         pts += frametime * 4;
 404       }
 405
 406       // Waiting timed out, output last picture
 407       if (m_picture.videoBuffer)
 408       {
 409         m_picture.pts = pts;
 410         m_outputSate = OutputPicture(&m_picture);
 411         pts += frametime;
 412       }
 413
 414       continue;
 415     }
 416
 417     if (pMsg->IsType(CDVDMsg::GENERAL_SYNCHRONIZE))
 418     {
 419       if (std::static_pointer_cast<CDVDMsgGeneralSynchronize>(pMsg)->Wait(100ms, SYNCSOURCE_VIDEO))
 420       {
 421         CLog::Log(LOGDEBUG, "CVideoPlayerVideo - CDVDMsg::GENERAL_SYNCHRONIZE");
 422       }
 423       else
 424         SendMessage(pMsg, 1); /* push back as prio message, to process other prio messages */
 425       m_droppingStats.Reset();
 426     }
 427     else if (pMsg->IsType(CDVDMsg::GENERAL_RESYNC))
 428     {
 429       pts = std::static_pointer_cast<CDVDMsgDouble>(pMsg)->m_value;
 430
 431       m_syncState = IDVDStreamPlayer::SYNC_INSYNC;
 432       m_droppingStats.Reset();
 433       m_rewindStalled = false;
 434       m_renderManager.ShowVideo(true);
 435
 436       CLog::Log(LOGDEBUG, "CVideoPlayerVideo - CDVDMsg::GENERAL_RESYNC({:f})", pts);
 437     }
 438     else if (pMsg->IsType(CDVDMsg::VIDEO_SET_ASPECT))
 439     {
 440       CLog::Log(LOGDEBUG, "CVideoPlayerVideo - CDVDMsg::VIDEO_SET_ASPECT");
 441       m_fForcedAspectRatio = static_cast<float>(*std::static_pointer_cast<CDVDMsgDouble>(pMsg));
 442     }
 443     else if (pMsg->IsType(CDVDMsg::GENERAL_RESET))
 444     {
 445       if(m_pVideoCodec)
 446         m_pVideoCodec->Reset();
 447
 448       if (m_picture.videoBuffer)
 449       {
 450         m_picture.videoBuffer->Release();
 451         m_picture.videoBuffer = nullptr;
 452       }
 453       m_packets.clear();
 454       m_droppingStats.Reset();
 455       m_syncState = IDVDStreamPlayer::SYNC_STARTING;
 456       m_renderManager.ShowVideo(false);
 457       m_rewindStalled = false;
 458     }
 459     else if (pMsg->IsType(CDVDMsg::GENERAL_FLUSH)) // private message sent by (CVideoPlayerVideo::Flush())
 460     {
 461       bool sync = std::static_pointer_cast<CDVDMsgBool>(pMsg)->m_value;
 462       if(m_pVideoCodec)
 463         m_pVideoCodec->Reset();
 464
 465       if (m_picture.videoBuffer)
 466       {
 467         m_picture.videoBuffer->Release();
 468         m_picture.videoBuffer = nullptr;
 469       }
 470       m_packets.clear();
 471       pts = 0;
 472       m_rewindStalled = false;
 473
 474       m_ptsTracker.Flush();
 475       //we need to recalculate the framerate
 476       //! @todo this needs to be set on a streamchange instead
 477       ResetFrameRateCalc();
 478       m_droppingStats.Reset();
 479
 480       m_stalled = true;
 481       if (sync)
 482       {
 483         m_syncState = IDVDStreamPlayer::SYNC_STARTING;
 484         m_renderManager.ShowVideo(false);
 485       }
 486
 487       m_renderManager.DiscardBuffer();
 488       FlushMessages();
 489     }
 490     else if (pMsg->IsType(CDVDMsg::PLAYER_SETSPEED))
 491     {
 492       m_speed = std::static_pointer_cast<CDVDMsgInt>(pMsg)->m_value;
 493       if (m_pVideoCodec)
 494         m_pVideoCodec->SetSpeed(m_speed);
 495
 496       m_droppingStats.Reset();
 497     }
 498     else if (pMsg->IsType(CDVDMsg::GENERAL_STREAMCHANGE))
 499     {
 500       auto msg = std::static_pointer_cast<CDVDMsgVideoCodecChange>(pMsg);
 501
 502       while (!m_bStop && m_pVideoCodec)
 503       {
 504         m_pVideoCodec->SetCodecControl(DVD_CODEC_CTRL_DRAIN);
 505         bool cont = ProcessDecoderOutput(frametime, pts);
 506
 507         if (!cont)
 508           break;
 509       }
 510
 511       OpenStream(msg->m_hints, std::move(msg->m_codec));
 512       msg->m_codec = NULL;
 513       if (m_picture.videoBuffer)
 514       {
 515         m_picture.videoBuffer->Release();
 516         m_picture.videoBuffer = nullptr;
 517       }
 518     }
 519     else if (pMsg->IsType(CDVDMsg::VIDEO_DRAIN))
 520     {
 521       while (!m_bStop && m_pVideoCodec)
 522       {
 523         m_pVideoCodec->SetCodecControl(DVD_CODEC_CTRL_DRAIN);
 524         if (!ProcessDecoderOutput(frametime, pts))
 525           break;
 526       }
 527     }
 528     else if (pMsg->IsType(CDVDMsg::GENERAL_PAUSE))
 529     {
 530       m_paused = std::static_pointer_cast<CDVDMsgBool>(pMsg)->m_value;
 531       CLog::Log(LOGDEBUG, "CVideoPlayerVideo - CDVDMsg::GENERAL_PAUSE: {}", m_paused);
 532     }
 533     else if (pMsg->IsType(CDVDMsg::PLAYER_REQUEST_STATE))
 534     {
 535       SStateMsg msg;
 536       msg.player = VideoPlayer_VIDEO;
 537       msg.syncState = m_syncState;
 538       m_messageParent.Put(
 539           std::make_shared<CDVDMsgType<SStateMsg>>(CDVDMsg::PLAYER_REPORT_STATE, msg));
 540     }
 541     else if (pMsg->IsType(CDVDMsg::DEMUXER_PACKET))
 542     {
 543       DemuxPacket* pPacket = std::static_pointer_cast<CDVDMsgDemuxerPacket>(pMsg)->GetPacket();
 544       bool bPacketDrop = std::static_pointer_cast<CDVDMsgDemuxerPacket>(pMsg)->GetPacketDrop();
 545
 546       if (m_stalled)
 547       {
 548         CLog::Log(LOGDEBUG, "CVideoPlayerVideo - Stillframe left, switching to normal playback");
 549         m_stalled = false;
 550       }
 551
 552       bRequestDrop = false;
 553       iDropDirective = CalcDropRequirement(pts);
 554       if ((iDropDirective & DROP_VERYLATE) &&
 555            m_bAllowDrop &&
 556           !bPacketDrop)
 557       {
 558         bRequestDrop = true;
 559       }
 560       if (iDropDirective & DROP_DROPPED)
 561       {
 562         m_iDroppedFrames++;
 563         m_ptsTracker.Flush();
 564       }
 565       if (m_messageQueue.GetDataSize() == 0 ||  m_speed < 0)
 566       {
 567         bRequestDrop = false;
 568         m_iDroppedRequest = 0;
 569         m_iLateFrames = 0;
 570       }
 571
 572       int codecControl = 0;
 573       if (iDropDirective & DROP_BUFFER_LEVEL)
 574         codecControl |= DVD_CODEC_CTRL_HURRY;
 575       if (m_speed > DVD_PLAYSPEED_NORMAL)
 576         codecControl |= DVD_CODEC_CTRL_NO_POSTPROC;
 577       if (bPacketDrop)
 578         codecControl |= DVD_CODEC_CTRL_DROP;
 579       if (bRequestDrop)
 580         codecControl |= DVD_CODEC_CTRL_DROP_ANY;
 581       if (!m_renderManager.Supports(RENDERFEATURE_ROTATION))
 582         codecControl |= DVD_CODEC_CTRL_ROTATE;
 583       m_pVideoCodec->SetCodecControl(codecControl);
 584
 585       if (m_pVideoCodec->AddData(*pPacket))
 586       {
 587         // buffer packets so we can recover should decoder flush for some reason
 588         if (m_pVideoCodec->GetConvergeCount() > 0)
 589         {
 590           m_packets.emplace_back(pMsg, 0);
 591           if (m_packets.size() > m_pVideoCodec->GetConvergeCount() ||
 592               m_packets.size() * frametime > DVD_SEC_TO_TIME(10))
 593             m_packets.pop_front();
 594         }
 595
 596         m_videoStats.AddSampleBytes(pPacket->iSize);
 597
 598         if (ProcessDecoderOutput(frametime, pts))
 599         {
 600           onlyPrioMsgs = true;
 601         }
 602       }
 603       else
 604       {
 605         SendMessageBack(pMsg);
 606         onlyPrioMsgs = true;
 607       }
 608     }
 609   }
 610 }
 611
 612 bool CVideoPlayerVideo::ProcessDecoderOutput(double &frametime, double &pts)
 613 {
 614   CDVDVideoCodec::VCReturn decoderState = m_pVideoCodec->GetPicture(&m_picture);
 615
 616   if (decoderState == CDVDVideoCodec::VC_BUFFER)
 617   {
 618     return false;
 619   }
 620
 621   // if decoder was flushed, we need to seek back again to resume rendering
 622   if (decoderState == CDVDVideoCodec::VC_FLUSHED)
 623   {
 624     CLog::Log(LOGDEBUG, "CVideoPlayerVideo - video decoder was flushed");
 625     while (!m_packets.empty())
 626     {
 627       auto msg = std::static_pointer_cast<CDVDMsgDemuxerPacket>(m_packets.front().message);
 628       m_packets.pop_front();
 629
 630       SendMessage(msg, 10);
 631     }
 632
 633     m_pVideoCodec->Reset();
 634     m_packets.clear();
 635     //picture.iFlags &= ~DVP_FLAG_ALLOCATED;
 636     m_renderManager.DiscardBuffer();
 637     return false;
 638   }
 639
 640   if (decoderState == CDVDVideoCodec::VC_REOPEN)
 641   {
 642     while (!m_packets.empty())
 643     {
 644       auto msg = std::static_pointer_cast<CDVDMsgDemuxerPacket>(m_packets.front().message);
 645       m_packets.pop_front();
 646       SendMessage(msg, 10);
 647     }
 648
 649     m_pVideoCodec->Reopen();
 650     m_packets.clear();
 651     m_renderManager.DiscardBuffer();
 652     return false;
 653   }
 654
 655   // if decoder had an error, tell it to reset to avoid more problems
 656   if (decoderState == CDVDVideoCodec::VC_ERROR)
 657   {
 658     CLog::Log(LOGDEBUG, "CVideoPlayerVideo - video decoder returned error");
 659     return false;
 660   }
 661
 662   if (decoderState == CDVDVideoCodec::VC_EOF)
 663   {
 664     if (m_syncState == IDVDStreamPlayer::SYNC_STARTING)
 665     {
 666       SStartMsg msg;
 667       msg.player = VideoPlayer_VIDEO;
 668       msg.cachetime = DVD_MSEC_TO_TIME(50);
 669       msg.cachetotal = DVD_MSEC_TO_TIME(100);
 670       msg.timestamp = DVD_NOPTS_VALUE;
 671       m_messageParent.Put(std::make_shared<CDVDMsgType<SStartMsg>>(CDVDMsg::PLAYER_STARTED, msg));
 672     }
 673     return false;
 674   }
 675
 676   // check for a new picture
 677   if (decoderState == CDVDVideoCodec::VC_PICTURE)
 678   {
 679     bool hasTimestamp = true;
 680
 681     m_picture.iDuration = frametime;
 682
 683     // validate picture timing,
 684     // if both dts/pts invalid, use pts calculated from picture.iDuration
 685     // if pts invalid use dts, else use picture.pts as passed
 686     if (m_picture.dts == DVD_NOPTS_VALUE && m_picture.pts == DVD_NOPTS_VALUE)
 687     {
 688       m_picture.pts = pts;
 689       hasTimestamp = false;
 690     }
 691     else if (m_picture.pts == DVD_NOPTS_VALUE)
 692       m_picture.pts = m_picture.dts;
 693
 694     // use forced aspect if any
 695     if (m_fForcedAspectRatio != 0.0f)
 696     {
 697       m_picture.iDisplayWidth = (int) (m_picture.iDisplayHeight * m_fForcedAspectRatio);
 698       if (m_picture.iDisplayWidth > m_picture.iWidth)
 699       {
 700         m_picture.iDisplayWidth =  m_picture.iWidth;
 701         m_picture.iDisplayHeight = (int) (m_picture.iDisplayWidth / m_fForcedAspectRatio);
 702       }
 703     }
 704
 705     // set stereo mode if not set by decoder
 706     if (m_picture.stereoMode.empty())
 707     {
 708       std::string stereoMode;
 709       switch(m_processInfo.GetVideoSettings().m_StereoMode)
 710       {
 711         case RENDER_STEREO_MODE_SPLIT_VERTICAL:
 712           stereoMode = "left_right";
 713           if (m_processInfo.GetVideoSettings().m_StereoInvert)
 714             stereoMode = "right_left";
 715           break;
 716         case RENDER_STEREO_MODE_SPLIT_HORIZONTAL:
 717           stereoMode = "top_bottom";
 718           if (m_processInfo.GetVideoSettings().m_StereoInvert)
 719             stereoMode = "bottom_top";
 720           break;
 721         default:
 722           stereoMode = m_hints.stereo_mode;
 723           break;
 724       }
 725       if (!stereoMode.empty() && stereoMode != "mono")
 726       {
 727         m_picture.stereoMode = stereoMode;
 728       }
 729     }
 730
 731     // if frame has a pts (usually originating from demux packet), use that
 732     if (m_picture.pts != DVD_NOPTS_VALUE)
 733     {
 734       pts = m_picture.pts;
 735     }
 736
 737     double extraDelay = 0.0;
 738     if (m_picture.iRepeatPicture)
 739     {
 740       extraDelay = m_picture.iRepeatPicture * m_picture.iDuration;
 741       m_picture.iDuration += extraDelay;
 742     }
 743
 744     m_picture.pts = pts + extraDelay;
 745     // guess next frame pts. iDuration is always valid
 746     if (m_speed != 0)
 747       pts += m_picture.iDuration * m_speed / abs(m_speed);
 748
 749     m_outputSate = OutputPicture(&m_picture);
 750
 751     if (m_outputSate == OUTPUT_AGAIN)
 752     {
 753       return true;
 754     }
 755     else if (m_outputSate == OUTPUT_ABORT)
 756     {
 757       return false;
 758     }
 759     else if ((m_outputSate == OUTPUT_DROPPED) && !(m_picture.iFlags & DVP_FLAG_DROPPED))
 760     {
 761       m_iDroppedFrames++;
 762       m_ptsTracker.Flush();
 763     }
 764
 765     if (m_syncState == IDVDStreamPlayer::SYNC_STARTING &&
 766         m_outputSate != OUTPUT_DROPPED &&
 767         !(m_picture.iFlags & DVP_FLAG_DROPPED))
 768     {
 769       m_syncState = IDVDStreamPlayer::SYNC_WAITSYNC;
 770       SStartMsg msg;
 771       msg.player = VideoPlayer_VIDEO;
 772       msg.cachetime = DVD_MSEC_TO_TIME(50); //! @todo implement
 773       msg.cachetotal = DVD_MSEC_TO_TIME(100); //! @todo implement
 774       msg.timestamp = hasTimestamp ? (pts + m_renderManager.GetDelay() * 1000) : DVD_NOPTS_VALUE;
 775       m_messageParent.Put(std::make_shared<CDVDMsgType<SStartMsg>>(CDVDMsg::PLAYER_STARTED, msg));
 776     }
 777
 778     frametime = (double)DVD_TIME_BASE / m_fFrameRate;
 779   }
 780
 781   return true;
 782 }
 783
 784 void CVideoPlayerVideo::OnExit()
 785 {
 786   CLog::Log(LOGINFO, "thread end: video_thread");
 787 }
 788
 789 void CVideoPlayerVideo::SetSpeed(int speed)
 790 {
 791   if(m_messageQueue.IsInited())
 792     SendMessage(std::make_shared<CDVDMsgInt>(CDVDMsg::PLAYER_SETSPEED, speed), 1);
 793   else
 794     m_speed = speed;
 795 }
 796
 797 void CVideoPlayerVideo::Flush(bool sync)
 798 {
 799   /* flush using message as this get's called from VideoPlayer thread */
 800   /* and any demux packet that has been taken out of queue need to */
 801   /* be disposed of before we flush */
 802   SendMessage(std::make_shared<CDVDMsgBool>(CDVDMsg::GENERAL_FLUSH, sync), 1);
 803   m_bAbortOutput = true;
 804 }
 805
 806 void CVideoPlayerVideo::ProcessOverlays(const VideoPicture* pSource, double pts)
 807 {
 808   // remove any overlays that are out of time
 809   if (m_syncState == IDVDStreamPlayer::SYNC_INSYNC)
 810     m_pOverlayContainer->CleanUp(pts - m_iSubtitleDelay);
 811
 812   VecOverlays overlays;
 813
 814   {
 815     std::unique_lock<CCriticalSection> lock(*m_pOverlayContainer);
 816
 817     VecOverlays* pVecOverlays = m_pOverlayContainer->GetOverlays();
 818     auto it = pVecOverlays->begin();
 819
 820     //Check all overlays and render those that should be rendered, based on time and forced
 821     //Both forced and subs should check timing
 822     while (it != pVecOverlays->end())
 823     {
 824       std::shared_ptr<CDVDOverlay>& pOverlay = *it++;
 825       if(!pOverlay->bForced && !m_bRenderSubs)
 826         continue;
 827
 828       double pts2 = pOverlay->bForced ? pts : pts - m_iSubtitleDelay;
 829
 830       if((pOverlay->iPTSStartTime <= pts2 && (pOverlay->iPTSStopTime > pts2 || pOverlay->iPTSStopTime == 0LL)))
 831       {
 832         if(pOverlay->IsOverlayType(DVDOVERLAY_TYPE_GROUP))
 833           overlays.insert(overlays.end(),
 834                           static_cast<CDVDOverlayGroup&>(*pOverlay).m_overlays.begin(),
 835                           static_cast<CDVDOverlayGroup&>(*pOverlay).m_overlays.end());
 836         else
 837           overlays.push_back(pOverlay);
 838       }
 839     }
 840
 841     for(it = overlays.begin(); it != overlays.end(); ++it)
 842     {
 843       double pts2 = (*it)->bForced ? pts : pts - m_iSubtitleDelay;
 844       m_renderManager.AddOverlay(*it, pts2);
 845     }
 846   }
 847 }
 848
 849 CVideoPlayerVideo::EOutputState CVideoPlayerVideo::OutputPicture(const VideoPicture* pPicture)
 850 {
 851   m_bAbortOutput = false;
 852
 853   if (m_processInfo.GetVideoStereoMode() != pPicture->stereoMode)
 854   {
 855     m_processInfo.SetVideoStereoMode(pPicture->stereoMode);
 856     // signal about changes in video parameters
 857     m_messageParent.Put(std::make_shared<CDVDMsg>(CDVDMsg::PLAYER_AVCHANGE));
 858   }
 859
 860   double config_framerate = m_bFpsInvalid ? 0.0 : m_fFrameRate;
 861   if (m_processInfo.GetVideoInterlaced())
 862   {
 863     if (MathUtils::FloatEquals(config_framerate, 25.0, 0.02))
 864       config_framerate = 50.0;
 865     else if (MathUtils::FloatEquals(config_framerate, 29.97, 0.02))
 866       config_framerate = 59.94;
 867   }
 868
 869   int sorient = m_processInfo.GetVideoSettings().m_Orientation;
 870   int orientation = sorient != 0 ? (sorient + m_hints.orientation) % 360
 871                                  : m_hints.orientation;
 872
 873   if (!m_renderManager.Configure(*pPicture,
 874                                 static_cast<float>(config_framerate),
 875                                 orientation,
 876                                 m_pVideoCodec->GetAllowedReferences()))
 877   {
 878     CLog::Log(LOGERROR, "{} - failed to configure renderer", __FUNCTION__);
 879     return OUTPUT_ABORT;
 880   }
 881
 882   //try to calculate the framerate
 883   m_ptsTracker.Add(pPicture->pts);
 884   if (!m_stalled)
 885     CalcFrameRate();
 886
 887   // signal to clock what our framerate is, it may want to adjust it's
 888   // speed to better match with our video renderer's output speed
 889   m_pClock->UpdateFramerate(m_fFrameRate);
 890
 891   // calculate the time we need to delay this picture before displaying
 892   double iPlayingClock, iCurrentClock;
 893
 894   iPlayingClock = m_pClock->GetClock(iCurrentClock, false); // snapshot current clock
 895
 896   if (m_speed < 0)
 897   {
 898     double renderPts;
 899     int queued, discard;
 900     int lateframes;
 901     double inputPts = m_droppingStats.m_lastPts;
 902     m_renderManager.GetStats(lateframes, renderPts, queued, discard);
 903     if (pPicture->pts > renderPts || queued > 0)
 904     {
 905       if (inputPts >= renderPts)
 906       {
 907         m_rewindStalled = true;
 908         CThread::Sleep(50ms);
 909       }
 910       return OUTPUT_DROPPED;
 911     }
 912     else if (pPicture->pts < iPlayingClock)
 913     {
 914       return OUTPUT_DROPPED;
 915     }
 916   }
 917
 918   if ((pPicture->iFlags & DVP_FLAG_DROPPED))
 919   {
 920     m_droppingStats.AddOutputDropGain(pPicture->pts, 1);
 921     CLog::Log(LOGDEBUG, "{} - dropped in output", __FUNCTION__);
 922     return OUTPUT_DROPPED;
 923   }
 924
 925   auto timeToDisplay = std::chrono::milliseconds(DVD_TIME_TO_MSEC(pPicture->pts - iPlayingClock));
 926
 927   // make sure waiting time is not negative
 928   std::chrono::milliseconds maxWaitTime = std::min(std::max(timeToDisplay + 500ms, 50ms), 500ms);
 929   // don't wait when going ff
 930   if (m_speed > DVD_PLAYSPEED_NORMAL)
 931     maxWaitTime = std::max(timeToDisplay, 0ms);
 932   int buffer = m_renderManager.WaitForBuffer(m_bAbortOutput, maxWaitTime);
 933   if (buffer < 0)
 934   {
 935     if (m_speed != DVD_PLAYSPEED_PAUSE)
 936       CLog::Log(LOGWARNING, "{} - timeout waiting for buffer", __FUNCTION__);
 937     return OUTPUT_AGAIN;
 938   }
 939
 940   ProcessOverlays(pPicture, pPicture->pts);
 941
 942   EINTERLACEMETHOD deintMethod = EINTERLACEMETHOD::VS_INTERLACEMETHOD_NONE;
 943   deintMethod = m_processInfo.GetVideoSettings().m_InterlaceMethod;
 944   if (!m_processInfo.Supports(deintMethod))
 945     deintMethod = m_processInfo.GetDeinterlacingMethodDefault();
 946
 947   if (!m_renderManager.AddVideoPicture(*pPicture, m_bAbortOutput, deintMethod, (m_syncState == ESyncState::SYNC_STARTING)))
 948   {
 949     m_droppingStats.AddOutputDropGain(pPicture->pts, 1);
 950     return OUTPUT_DROPPED;
 951   }
 952
 953   return OUTPUT_NORMAL;
 954 }
 955
 956 std::string CVideoPlayerVideo::GetPlayerInfo()
 957 {
 958   std::ostringstream s;
 959   s << "vq:" << std::setw(2) << std::min(99, m_processInfo.GetLevelVQ());
 960   s << "% " << std::fixed << std::setprecision(3) << m_messageQueue.GetTimeSize();
 961   s << "s, Mb/s:" << std::fixed << std::setprecision(2)
 962     << static_cast<double>(GetVideoBitrate()) / (1024.0 * 1024.0);
 963   s << ", fr:" << std::fixed << std::setprecision(3) << m_fFrameRate;
 964   s << ", drop:" << m_iDroppedFrames;
 965   s << ", skip:" << m_renderManager.GetSkippedFrames();
 966
 967   int pc = m_ptsTracker.GetPatternLength();
 968   if (pc > 0)
 969     s << ", pc:" << pc;
 970   else
 971     s << ", pc:none";
 972
 973   return s.str();
 974 }
 975
 976 int CVideoPlayerVideo::GetVideoBitrate()
 977 {
 978   return (int)m_videoStats.GetBitrate();
 979 }
 980
 981 void CVideoPlayerVideo::ResetFrameRateCalc()
 982 {
 983   m_fStableFrameRate = 0.0;
 984   m_iFrameRateCount = 0;
 985   m_iFrameRateLength = 1;
 986   m_iFrameRateErr = 0;
 987   m_bAllowDrop = CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_videoFpsDetect == 0;
 988 }
 989
 990 double CVideoPlayerVideo::GetCurrentPts()
 991 {
 992   double renderPts;
 993   int sleepTime;
 994   int queued, discard;
 995
 996   // get render stats
 997   m_renderManager.GetStats(sleepTime, renderPts, queued, discard);
 998
 999   if (renderPts == DVD_NOPTS_VALUE)
1000     return DVD_NOPTS_VALUE;
1001   else if (m_stalled)
1002     return DVD_NOPTS_VALUE;
1003   else if (m_speed == DVD_PLAYSPEED_NORMAL)
1004   {
1005     if (renderPts < 0)
1006       renderPts = 0;
1007   }
1008   return renderPts;
1009 }
1010
1011 #define MAXFRAMERATEDIFF   0.01
1012 #define MAXFRAMESERR    1000
1013
1014 void CVideoPlayerVideo::CalcFrameRate()
1015 {
1016   if (m_iFrameRateLength >= 128 || CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_videoFpsDetect == 0)
1017     return; //don't calculate the fps
1018
1019   if (!m_ptsTracker.HasFullBuffer())
1020     return; //we can only calculate the frameduration if m_pullupCorrection has a full buffer
1021
1022   //see if m_pullupCorrection was able to detect a pattern in the timestamps
1023   //and is able to calculate the correct frame duration from it
1024   double frameduration = m_ptsTracker.GetFrameDuration();
1025   if (m_ptsTracker.VFRDetection())
1026     frameduration = m_ptsTracker.GetMinFrameDuration();
1027
1028   if ((frameduration==DVD_NOPTS_VALUE) ||
1029       ((CServiceBroker::GetSettingsComponent()->GetAdvancedSettings()->m_videoFpsDetect == 1) && ((m_ptsTracker.GetPatternLength() > 1) && !m_ptsTracker.VFRDetection())))
1030   {
1031     //reset the stored framerates if no good framerate was detected
1032     m_fStableFrameRate = 0.0;
1033     m_iFrameRateCount = 0;
1034     m_iFrameRateErr++;
1035
1036     if (m_iFrameRateErr == MAXFRAMESERR && m_iFrameRateLength == 1)
1037     {
1038       CLog::Log(LOGDEBUG,
1039                 "{} counted {} frames without being able to calculate the framerate, giving up",
1040                 __FUNCTION__, m_iFrameRateErr);
1041       m_bAllowDrop = true;
1042       m_iFrameRateLength = 128;
1043     }
1044     return;
1045   }
1046
1047   double framerate = DVD_TIME_BASE / frameduration;
1048
1049   //store the current calculated framerate if we don't have any yet
1050   if (m_iFrameRateCount == 0)
1051   {
1052     m_fStableFrameRate = framerate;
1053     m_iFrameRateCount++;
1054   }
1055   //check if the current detected framerate matches with the stored ones
1056   else if (fabs(m_fStableFrameRate / m_iFrameRateCount - framerate) <= MAXFRAMERATEDIFF)
1057   {
1058     m_fStableFrameRate += framerate; //store the calculated framerate
1059     m_iFrameRateCount++;
1060
1061     //if we've measured m_iFrameRateLength seconds of framerates,
1062     if (m_iFrameRateCount >= MathUtils::round_int(framerate) * m_iFrameRateLength)
1063     {
1064       //store the calculated framerate if it differs too much from m_fFrameRate
1065       if (fabs(m_fFrameRate - (m_fStableFrameRate / m_iFrameRateCount)) > MAXFRAMERATEDIFF || m_bFpsInvalid)
1066       {
1067         CLog::Log(LOGDEBUG, "{} framerate was:{:f} calculated:{:f}", __FUNCTION__, m_fFrameRate,
1068                   m_fStableFrameRate / m_iFrameRateCount);
1069         m_fFrameRate = m_fStableFrameRate / m_iFrameRateCount;
1070         m_bFpsInvalid = false;
1071         m_processInfo.SetVideoFps(static_cast<float>(m_fFrameRate));
1072       }
1073
1074       //reset the stored framerates
1075       m_fStableFrameRate = 0.0;
1076       m_iFrameRateCount = 0;
1077       m_iFrameRateLength *= 2; //double the length we should measure framerates
1078
1079       //we're allowed to drop frames because we calculated a good framerate
1080       m_bAllowDrop = true;
1081     }
1082   }
1083   else //the calculated framerate didn't match, reset the stored ones
1084   {
1085     m_fStableFrameRate = 0.0;
1086     m_iFrameRateCount = 0;
1087   }
1088 }
1089
1090 int CVideoPlayerVideo::CalcDropRequirement(double pts)
1091 {
1092   int result = 0;
1093   int lateframes;
1094   double iDecoderPts, iRenderPts;
1095   int iSkippedPicture = -1;
1096   int iDroppedFrames = -1;
1097   int iBufferLevel;
1098   int queued, discard;
1099
1100   m_droppingStats.m_lastPts = pts;
1101
1102   // get decoder stats
1103   if (!m_pVideoCodec->GetCodecStats(iDecoderPts, iDroppedFrames, iSkippedPicture))
1104     iDecoderPts = pts;
1105   if (iDecoderPts == DVD_NOPTS_VALUE)
1106     iDecoderPts = pts;
1107
1108   // get render stats
1109   m_renderManager.GetStats(lateframes, iRenderPts, queued, discard);
1110   iBufferLevel = queued + discard;
1111
1112   if (iBufferLevel < 0)
1113     result |= DROP_BUFFER_LEVEL;
1114   else if (iBufferLevel < 2)
1115   {
1116     result |= DROP_BUFFER_LEVEL;
1117     CLog::Log(LOGDEBUG, LOGVIDEO, "CVideoPlayerVideo::CalcDropRequirement - hurry: {}",
1118               iBufferLevel);
1119   }
1120
1121   if (m_bAllowDrop)
1122   {
1123     if (iSkippedPicture > 0)
1124     {
1125       CDroppingStats::CGain gain;
1126       gain.frames = iSkippedPicture;
1127       gain.pts = iDecoderPts;
1128       m_droppingStats.m_gain.push_back(gain);
1129       m_droppingStats.m_totalGain += gain.frames;
1130       result |= DROP_DROPPED;
1131       CLog::Log(LOGDEBUG, LOGVIDEO,
1132                 "CVideoPlayerVideo::CalcDropRequirement - dropped pictures, lateframes: {}, "
1133                 "Bufferlevel: {}, dropped: {}",
1134                 lateframes, iBufferLevel, iSkippedPicture);
1135     }
1136     if (iDroppedFrames > 0)
1137     {
1138       CDroppingStats::CGain gain;
1139       gain.frames = iDroppedFrames;
1140       gain.pts = iDecoderPts;
1141       m_droppingStats.m_gain.push_back(gain);
1142       m_droppingStats.m_totalGain += iDroppedFrames;
1143       result |= DROP_DROPPED;
1144       CLog::Log(LOGDEBUG, LOGVIDEO,
1145                 "CVideoPlayerVideo::CalcDropRequirement - dropped in decoder, lateframes: {}, "
1146                 "Bufferlevel: {}, dropped: {}",
1147                 lateframes, iBufferLevel, iDroppedFrames);
1148     }
1149   }
1150
1151   // subtract gains
1152   while (!m_droppingStats.m_gain.empty() &&
1153          iRenderPts >= m_droppingStats.m_gain.front().pts)
1154   {
1155     m_droppingStats.m_totalGain -= m_droppingStats.m_gain.front().frames;
1156     m_droppingStats.m_gain.pop_front();
1157   }
1158
1159   // calculate lateness
1160   int lateness = lateframes - m_droppingStats.m_totalGain;
1161
1162   if (lateness > 0 && m_speed)
1163   {
1164     result |= DROP_VERYLATE;
1165   }
1166   return result;
1167 }
1168
1169 void CDroppingStats::Reset()
1170 {
1171   m_gain.clear();
1172   m_totalGain = 0;
1173 }
1174
1175 void CDroppingStats::AddOutputDropGain(double pts, int frames)
1176 {
1177   CDroppingStats::CGain gain;
1178   gain.frames = frames;
1179   gain.pts = pts;
1180   m_gain.push_back(gain);
1181   m_totalGain += frames;
1182 }