media/filters/video_renderer_impl.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/video_renderer_impl.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback.h"
   9 #include "base/callback_helpers.h"
  10 #include "base/debug/trace_event.h"
  11 #include "base/location.h"
  12 #include "base/single_thread_task_runner.h"
  13 #include "base/threading/platform_thread.h"
  14 #include "media/base/buffers.h"
  15 #include "media/base/limits.h"
  16 #include "media/base/pipeline.h"
  17 #include "media/base/video_frame.h"
  18
  19 namespace media {
  20
  21 VideoRendererImpl::VideoRendererImpl(
  22     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
  23     ScopedVector<VideoDecoder> decoders,
  24     const SetDecryptorReadyCB& set_decryptor_ready_cb,
  25     const PaintCB& paint_cb,
  26     bool drop_frames)
  27     : task_runner_(task_runner),
  28       video_frame_stream_(task_runner, decoders.Pass(), set_decryptor_ready_cb),
  29       low_delay_(false),
  30       received_end_of_stream_(false),
  31       rendered_end_of_stream_(false),
  32       frame_available_(&lock_),
  33       state_(kUninitialized),
  34       thread_(),
  35       pending_read_(false),
  36       drop_frames_(drop_frames),
  37       playback_rate_(0),
  38       paint_cb_(paint_cb),
  39       last_timestamp_(kNoTimestamp()),
  40       frames_decoded_(0),
  41       frames_dropped_(0),
  42       weak_factory_(this) {
  43   DCHECK(!paint_cb_.is_null());
  44 }
  45
  46 VideoRendererImpl::~VideoRendererImpl() {
  47   base::AutoLock auto_lock(lock_);
  48   CHECK(state_ == kStopped || state_ == kUninitialized) << state_;
  49   CHECK(thread_.is_null());
  50 }
  51
  52 void VideoRendererImpl::Play(const base::Closure& callback) {
  53   DCHECK(task_runner_->BelongsToCurrentThread());
  54   base::AutoLock auto_lock(lock_);
  55   DCHECK_EQ(kPrerolled, state_);
  56   state_ = kPlaying;
  57   callback.Run();
  58 }
  59
  60 void VideoRendererImpl::Flush(const base::Closure& callback) {
  61   DCHECK(task_runner_->BelongsToCurrentThread());
  62   base::AutoLock auto_lock(lock_);
  63   DCHECK_NE(state_, kUninitialized);
  64   flush_cb_ = callback;
  65   state_ = kFlushing;
  66
  67   // This is necessary if the |video_frame_stream_| has already seen an end of
  68   // stream and needs to drain it before flushing it.
  69   ready_frames_.clear();
  70   received_end_of_stream_ = false;
  71   rendered_end_of_stream_ = false;
  72   video_frame_stream_.Reset(
  73       base::Bind(&VideoRendererImpl::OnVideoFrameStreamResetDone,
  74                  weak_factory_.GetWeakPtr()));
  75 }
  76
  77 void VideoRendererImpl::Stop(const base::Closure& callback) {
  78   DCHECK(task_runner_->BelongsToCurrentThread());
  79   base::AutoLock auto_lock(lock_);
  80   if (state_ == kUninitialized || state_ == kStopped) {
  81     callback.Run();
  82     return;
  83   }
  84
  85   // TODO(scherkus): Consider invalidating |weak_factory_| and replacing
  86   // task-running guards that check |state_| with DCHECK().
  87
  88   state_ = kStopped;
  89
  90   statistics_cb_.Reset();
  91   max_time_cb_.Reset();
  92   DoStopOrError_Locked();
  93
  94   // Clean up our thread if present.
  95   base::PlatformThreadHandle thread_to_join = base::PlatformThreadHandle();
  96   if (!thread_.is_null()) {
  97     // Signal the thread since it's possible to get stopped with the video
  98     // thread waiting for a read to complete.
  99     frame_available_.Signal();
 100     std::swap(thread_, thread_to_join);
 101   }
 102
 103   if (!thread_to_join.is_null()) {
 104     base::AutoUnlock auto_unlock(lock_);
 105     base::PlatformThread::Join(thread_to_join);
 106   }
 107
 108   video_frame_stream_.Stop(callback);
 109 }
 110
 111 void VideoRendererImpl::SetPlaybackRate(float playback_rate) {
 112   DCHECK(task_runner_->BelongsToCurrentThread());
 113   base::AutoLock auto_lock(lock_);
 114   playback_rate_ = playback_rate;
 115 }
 116
 117 void VideoRendererImpl::Preroll(base::TimeDelta time,
 118                                 const PipelineStatusCB& cb) {
 119   DCHECK(task_runner_->BelongsToCurrentThread());
 120   base::AutoLock auto_lock(lock_);
 121   DCHECK(!cb.is_null());
 122   DCHECK(preroll_cb_.is_null());
 123   DCHECK(state_ == kFlushed || state_ == kPlaying) << "state_ " << state_;
 124
 125   if (state_ == kFlushed) {
 126     DCHECK(time != kNoTimestamp());
 127     DCHECK(!pending_read_);
 128     DCHECK(ready_frames_.empty());
 129   } else {
 130     DCHECK(time == kNoTimestamp());
 131   }
 132
 133   state_ = kPrerolling;
 134   preroll_cb_ = cb;
 135   preroll_timestamp_ = time;
 136
 137   if (ShouldTransitionToPrerolled_Locked()) {
 138     TransitionToPrerolled_Locked();
 139     return;
 140   }
 141
 142   AttemptRead_Locked();
 143 }
 144
 145 void VideoRendererImpl::Initialize(DemuxerStream* stream,
 146                                    bool low_delay,
 147                                    const PipelineStatusCB& init_cb,
 148                                    const StatisticsCB& statistics_cb,
 149                                    const TimeCB& max_time_cb,
 150                                    const base::Closure& ended_cb,
 151                                    const PipelineStatusCB& error_cb,
 152                                    const TimeDeltaCB& get_time_cb,
 153                                    const TimeDeltaCB& get_duration_cb) {
 154   DCHECK(task_runner_->BelongsToCurrentThread());
 155   base::AutoLock auto_lock(lock_);
 156   DCHECK(stream);
 157   DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
 158   DCHECK(!init_cb.is_null());
 159   DCHECK(!statistics_cb.is_null());
 160   DCHECK(!max_time_cb.is_null());
 161   DCHECK(!ended_cb.is_null());
 162   DCHECK(!get_time_cb.is_null());
 163   DCHECK(!get_duration_cb.is_null());
 164   DCHECK_EQ(kUninitialized, state_);
 165
 166   low_delay_ = low_delay;
 167
 168   init_cb_ = init_cb;
 169   statistics_cb_ = statistics_cb;
 170   max_time_cb_ = max_time_cb;
 171   ended_cb_ = ended_cb;
 172   error_cb_ = error_cb;
 173   get_time_cb_ = get_time_cb;
 174   get_duration_cb_ = get_duration_cb;
 175   state_ = kInitializing;
 176
 177   video_frame_stream_.Initialize(
 178       stream,
 179       low_delay,
 180       statistics_cb,
 181       base::Bind(&VideoRendererImpl::OnVideoFrameStreamInitialized,
 182                  weak_factory_.GetWeakPtr()));
 183 }
 184
 185 void VideoRendererImpl::OnVideoFrameStreamInitialized(bool success) {
 186   DCHECK(task_runner_->BelongsToCurrentThread());
 187   base::AutoLock auto_lock(lock_);
 188
 189   if (state_ == kStopped)
 190     return;
 191
 192   DCHECK_EQ(state_, kInitializing);
 193
 194   if (!success) {
 195     state_ = kUninitialized;
 196     base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
 197     return;
 198   }
 199
 200   // We're all good!  Consider ourselves flushed. (ThreadMain() should never
 201   // see us in the kUninitialized state).
 202   // Since we had an initial Preroll(), we consider ourself flushed, because we
 203   // have not populated any buffers yet.
 204   state_ = kFlushed;
 205
 206   // Create our video thread.
 207   CHECK(base::PlatformThread::Create(0, this, &thread_));
 208
 209 #if defined(OS_WIN)
 210   // Bump up our priority so our sleeping is more accurate.
 211   // TODO(scherkus): find out if this is necessary, but it seems to help.
 212   ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL);
 213 #endif  // defined(OS_WIN)
 214   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
 215 }
 216
 217 // PlatformThread::Delegate implementation.
 218 void VideoRendererImpl::ThreadMain() {
 219   base::PlatformThread::SetName("CrVideoRenderer");
 220
 221   // The number of milliseconds to idle when we do not have anything to do.
 222   // Nothing special about the value, other than we're being more OS-friendly
 223   // than sleeping for 1 millisecond.
 224   //
 225   // TODO(scherkus): switch to pure event-driven frame timing instead of this
 226   // kIdleTimeDelta business http://crbug.com/106874
 227   const base::TimeDelta kIdleTimeDelta =
 228       base::TimeDelta::FromMilliseconds(10);
 229
 230   for (;;) {
 231     base::AutoLock auto_lock(lock_);
 232
 233     // Thread exit condition.
 234     if (state_ == kStopped)
 235       return;
 236
 237     // Remain idle as long as we're not playing.
 238     if (state_ != kPlaying || playback_rate_ == 0) {
 239       UpdateStatsAndWait_Locked(kIdleTimeDelta);
 240       continue;
 241     }
 242
 243     // Remain idle until we have the next frame ready for rendering.
 244     if (ready_frames_.empty()) {
 245       if (received_end_of_stream_ && !rendered_end_of_stream_) {
 246         rendered_end_of_stream_ = true;
 247         ended_cb_.Run();
 248       }
 249
 250       UpdateStatsAndWait_Locked(kIdleTimeDelta);
 251       continue;
 252     }
 253
 254     base::TimeDelta remaining_time =
 255         CalculateSleepDuration(ready_frames_.front(), playback_rate_);
 256
 257     // Sleep up to a maximum of our idle time until we're within the time to
 258     // render the next frame.
 259     if (remaining_time.InMicroseconds() > 0) {
 260       remaining_time = std::min(remaining_time, kIdleTimeDelta);
 261       UpdateStatsAndWait_Locked(remaining_time);
 262       continue;
 263     }
 264
 265     // Deadline is defined as the midpoint between this frame and the next
 266     // frame, using the delta between this frame and the previous frame as the
 267     // assumption for frame duration.
 268     //
 269     // TODO(scherkus): An improvement over midpoint might be selecting the
 270     // minimum and/or maximum between the midpoint and some constants. As a
 271     // thought experiment, consider what would be better than the midpoint
 272     // for both the 1fps case and 120fps case.
 273     //
 274     // TODO(scherkus): This can be vastly improved. Use a histogram to measure
 275     // the accuracy of our frame timing code. http://crbug.com/149829
 276     if (drop_frames_ && last_timestamp_ != kNoTimestamp()) {
 277       base::TimeDelta now = get_time_cb_.Run();
 278       base::TimeDelta deadline = ready_frames_.front()->timestamp() +
 279           (ready_frames_.front()->timestamp() - last_timestamp_) / 2;
 280
 281       if (now > deadline) {
 282         DropNextReadyFrame_Locked();
 283         continue;
 284       }
 285     }
 286
 287     // Congratulations! You've made it past the video frame timing gauntlet.
 288     //
 289     // At this point enough time has passed that the next frame that ready for
 290     // rendering.
 291     PaintNextReadyFrame_Locked();
 292   }
 293 }
 294
 295 void VideoRendererImpl::PaintNextReadyFrame_Locked() {
 296   lock_.AssertAcquired();
 297
 298   scoped_refptr<VideoFrame> next_frame = ready_frames_.front();
 299   ready_frames_.pop_front();
 300   frames_decoded_++;
 301
 302   last_timestamp_ = next_frame->timestamp();
 303
 304   paint_cb_.Run(next_frame);
 305
 306   task_runner_->PostTask(
 307       FROM_HERE,
 308       base::Bind(&VideoRendererImpl::AttemptRead, weak_factory_.GetWeakPtr()));
 309 }
 310
 311 void VideoRendererImpl::DropNextReadyFrame_Locked() {
 312   TRACE_EVENT0("media", "VideoRendererImpl:frameDropped");
 313
 314   lock_.AssertAcquired();
 315
 316   last_timestamp_ = ready_frames_.front()->timestamp();
 317   ready_frames_.pop_front();
 318   frames_decoded_++;
 319   frames_dropped_++;
 320
 321   task_runner_->PostTask(
 322       FROM_HERE,
 323       base::Bind(&VideoRendererImpl::AttemptRead, weak_factory_.GetWeakPtr()));
 324 }
 325
 326 void VideoRendererImpl::FrameReady(VideoFrameStream::Status status,
 327                                    const scoped_refptr<VideoFrame>& frame) {
 328   base::AutoLock auto_lock(lock_);
 329   DCHECK_NE(state_, kUninitialized);
 330   DCHECK_NE(state_, kFlushed);
 331
 332   CHECK(pending_read_);
 333   pending_read_ = false;
 334
 335   if (status == VideoFrameStream::DECODE_ERROR ||
 336       status == VideoFrameStream::DECRYPT_ERROR) {
 337     DCHECK(!frame.get());
 338     PipelineStatus error = PIPELINE_ERROR_DECODE;
 339     if (status == VideoFrameStream::DECRYPT_ERROR)
 340       error = PIPELINE_ERROR_DECRYPT;
 341
 342     if (!preroll_cb_.is_null()) {
 343       base::ResetAndReturn(&preroll_cb_).Run(error);
 344       return;
 345     }
 346
 347     error_cb_.Run(error);
 348     return;
 349   }
 350
 351   // Already-queued VideoFrameStream ReadCB's can fire after various state
 352   // transitions have happened; in that case just drop those frames immediately.
 353   if (state_ == kStopped || state_ == kFlushing)
 354     return;
 355
 356   if (!frame.get()) {
 357     // Abort preroll early for a NULL frame because we won't get more frames.
 358     // A new preroll will be requested after this one completes so there is no
 359     // point trying to collect more frames.
 360     if (state_ == kPrerolling)
 361       TransitionToPrerolled_Locked();
 362
 363     return;
 364   }
 365
 366   if (frame->end_of_stream()) {
 367     DCHECK(!received_end_of_stream_);
 368     received_end_of_stream_ = true;
 369     max_time_cb_.Run(get_duration_cb_.Run());
 370
 371     if (state_ == kPrerolling)
 372       TransitionToPrerolled_Locked();
 373
 374     return;
 375   }
 376
 377   // Maintain the latest frame decoded so the correct frame is displayed after
 378   // prerolling has completed.
 379   if (state_ == kPrerolling && preroll_timestamp_ != kNoTimestamp() &&
 380       frame->timestamp() <= preroll_timestamp_) {
 381     ready_frames_.clear();
 382   }
 383
 384   AddReadyFrame_Locked(frame);
 385
 386   if (ShouldTransitionToPrerolled_Locked())
 387     TransitionToPrerolled_Locked();
 388
 389   // Always request more decoded video if we have capacity. This serves two
 390   // purposes:
 391   //   1) Prerolling while paused
 392   //   2) Keeps decoding going if video rendering thread starts falling behind
 393   AttemptRead_Locked();
 394 }
 395
 396 bool VideoRendererImpl::ShouldTransitionToPrerolled_Locked() {
 397   return state_ == kPrerolling &&
 398       (!video_frame_stream_.CanReadWithoutStalling() ||
 399        ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames) ||
 400        (low_delay_ && ready_frames_.size() > 0));
 401 }
 402
 403 void VideoRendererImpl::AddReadyFrame_Locked(
 404     const scoped_refptr<VideoFrame>& frame) {
 405   lock_.AssertAcquired();
 406   DCHECK(!frame->end_of_stream());
 407
 408   // Adjust the incoming frame if its rendering stop time is past the duration
 409   // of the video itself. This is typically the last frame of the video and
 410   // occurs if the container specifies a duration that isn't a multiple of the
 411   // frame rate.  Another way for this to happen is for the container to state
 412   // a smaller duration than the largest packet timestamp.
 413   base::TimeDelta duration = get_duration_cb_.Run();
 414   if (frame->timestamp() > duration) {
 415     frame->set_timestamp(duration);
 416   }
 417
 418   ready_frames_.push_back(frame);
 419   DCHECK_LE(ready_frames_.size(),
 420             static_cast<size_t>(limits::kMaxVideoFrames));
 421
 422   max_time_cb_.Run(frame->timestamp());
 423
 424   // Avoid needlessly waking up |thread_| unless playing.
 425   if (state_ == kPlaying)
 426     frame_available_.Signal();
 427 }
 428
 429 void VideoRendererImpl::AttemptRead() {
 430   base::AutoLock auto_lock(lock_);
 431   AttemptRead_Locked();
 432 }
 433
 434 void VideoRendererImpl::AttemptRead_Locked() {
 435   DCHECK(task_runner_->BelongsToCurrentThread());
 436   lock_.AssertAcquired();
 437
 438   if (pending_read_ || received_end_of_stream_ ||
 439       ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) {
 440     return;
 441   }
 442
 443   switch (state_) {
 444     case kPrerolling:
 445     case kPrerolled:
 446     case kPlaying:
 447       pending_read_ = true;
 448       video_frame_stream_.Read(base::Bind(&VideoRendererImpl::FrameReady,
 449                                           weak_factory_.GetWeakPtr()));
 450       return;
 451
 452     case kUninitialized:
 453     case kInitializing:
 454     case kFlushing:
 455     case kFlushed:
 456     case kStopped:
 457       return;
 458   }
 459 }
 460
 461 void VideoRendererImpl::OnVideoFrameStreamResetDone() {
 462   base::AutoLock auto_lock(lock_);
 463   if (state_ == kStopped)
 464     return;
 465
 466   DCHECK_EQ(kFlushing, state_);
 467   DCHECK(!pending_read_);
 468   DCHECK(ready_frames_.empty());
 469   DCHECK(!received_end_of_stream_);
 470   DCHECK(!rendered_end_of_stream_);
 471
 472   state_ = kFlushed;
 473   last_timestamp_ = kNoTimestamp();
 474   base::ResetAndReturn(&flush_cb_).Run();
 475 }
 476
 477 base::TimeDelta VideoRendererImpl::CalculateSleepDuration(
 478     const scoped_refptr<VideoFrame>& next_frame,
 479     float playback_rate) {
 480   // Determine the current and next presentation timestamps.
 481   base::TimeDelta now = get_time_cb_.Run();
 482   base::TimeDelta next_pts = next_frame->timestamp();
 483
 484   // Scale our sleep based on the playback rate.
 485   base::TimeDelta sleep = next_pts - now;
 486   return base::TimeDelta::FromMicroseconds(
 487       static_cast<int64>(sleep.InMicroseconds() / playback_rate));
 488 }
 489
 490 void VideoRendererImpl::DoStopOrError_Locked() {
 491   lock_.AssertAcquired();
 492   last_timestamp_ = kNoTimestamp();
 493   ready_frames_.clear();
 494 }
 495
 496 void VideoRendererImpl::TransitionToPrerolled_Locked() {
 497   lock_.AssertAcquired();
 498   DCHECK_EQ(state_, kPrerolling);
 499
 500   state_ = kPrerolled;
 501
 502   // Because we might remain in the prerolled state for an undetermined amount
 503   // of time (e.g., we seeked while paused), we'll paint the first prerolled
 504   // frame.
 505   if (!ready_frames_.empty())
 506     PaintNextReadyFrame_Locked();
 507
 508   base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK);
 509 }
 510
 511 void VideoRendererImpl::UpdateStatsAndWait_Locked(
 512     base::TimeDelta wait_duration) {
 513   lock_.AssertAcquired();
 514   DCHECK_GE(frames_decoded_, 0);
 515   DCHECK_LE(frames_dropped_, frames_decoded_);
 516
 517   if (frames_decoded_) {
 518     PipelineStatistics statistics;
 519     statistics.video_frames_decoded = frames_decoded_;
 520     statistics.video_frames_dropped = frames_dropped_;
 521     statistics_cb_.Run(statistics);
 522
 523     frames_decoded_ = 0;
 524     frames_dropped_ = 0;
 525   }
 526
 527   frame_available_.TimedWait(wait_duration);
 528 }
 529
 530 }  // namespace media