media/filters/video_renderer_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/video_renderer_base.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback.h"
   9 #include "base/callback_helpers.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/threading/platform_thread.h"
  12 #include "media/base/buffers.h"
  13 #include "media/base/limits.h"
  14 #include "media/base/pipeline.h"
  15 #include "media/base/video_frame.h"
  16
  17 namespace media {
  18
  19 base::TimeDelta VideoRendererBase::kMaxLastFrameDuration() {
  20   return base::TimeDelta::FromMilliseconds(250);
  21 }
  22
  23 VideoRendererBase::VideoRendererBase(
  24     const scoped_refptr<base::MessageLoopProxy>& message_loop,
  25     ScopedVector<VideoDecoder> decoders,
  26     const SetDecryptorReadyCB& set_decryptor_ready_cb,
  27     const PaintCB& paint_cb,
  28     const SetOpaqueCB& set_opaque_cb,
  29     bool drop_frames)
  30     : message_loop_(message_loop),
  31       weak_factory_(this),
  32       video_frame_stream_(
  33           message_loop, decoders.Pass(), set_decryptor_ready_cb),
  34       received_end_of_stream_(false),
  35       frame_available_(&lock_),
  36       state_(kUninitialized),
  37       thread_(),
  38       pending_read_(false),
  39       drop_frames_(drop_frames),
  40       playback_rate_(0),
  41       paint_cb_(paint_cb),
  42       set_opaque_cb_(set_opaque_cb),
  43       last_timestamp_(kNoTimestamp()),
  44       frames_decoded_(0),
  45       frames_dropped_(0) {
  46   DCHECK(!paint_cb_.is_null());
  47 }
  48
  49 VideoRendererBase::~VideoRendererBase() {
  50   base::AutoLock auto_lock(lock_);
  51   CHECK(state_ == kStopped || state_ == kUninitialized) << state_;
  52   CHECK(thread_.is_null());
  53 }
  54
  55 void VideoRendererBase::Play(const base::Closure& callback) {
  56   DCHECK(message_loop_->BelongsToCurrentThread());
  57   base::AutoLock auto_lock(lock_);
  58   DCHECK_EQ(kPrerolled, state_);
  59   state_ = kPlaying;
  60   callback.Run();
  61 }
  62
  63 void VideoRendererBase::Pause(const base::Closure& callback) {
  64   DCHECK(message_loop_->BelongsToCurrentThread());
  65   base::AutoLock auto_lock(lock_);
  66   DCHECK(state_ != kUninitialized || state_ == kError);
  67   state_ = kPaused;
  68   callback.Run();
  69 }
  70
  71 void VideoRendererBase::Flush(const base::Closure& callback) {
  72   DCHECK(message_loop_->BelongsToCurrentThread());
  73   base::AutoLock auto_lock(lock_);
  74   DCHECK_EQ(state_, kPaused);
  75   flush_cb_ = callback;
  76   state_ = kFlushing;
  77
  78   // This is necessary if the |video_frame_stream_| has already seen an end of
  79   // stream and needs to drain it before flushing it.
  80   ready_frames_.clear();
  81   received_end_of_stream_ = false;
  82   video_frame_stream_.Reset(base::Bind(
  83       &VideoRendererBase::OnVideoFrameStreamResetDone, weak_this_));
  84 }
  85
  86 void VideoRendererBase::Stop(const base::Closure& callback) {
  87   DCHECK(message_loop_->BelongsToCurrentThread());
  88   base::AutoLock auto_lock(lock_);
  89   if (state_ == kUninitialized || state_ == kStopped) {
  90     callback.Run();
  91     return;
  92   }
  93
  94   // TODO(scherkus): Consider invalidating |weak_factory_| and replacing
  95   // task-running guards that check |state_| with DCHECK().
  96
  97   state_ = kStopped;
  98
  99   statistics_cb_.Reset();
 100   max_time_cb_.Reset();
 101   DoStopOrError_Locked();
 102
 103   // Clean up our thread if present.
 104   base::PlatformThreadHandle thread_to_join = base::PlatformThreadHandle();
 105   if (!thread_.is_null()) {
 106     // Signal the thread since it's possible to get stopped with the video
 107     // thread waiting for a read to complete.
 108     frame_available_.Signal();
 109     std::swap(thread_, thread_to_join);
 110   }
 111
 112   if (!thread_to_join.is_null()) {
 113     base::AutoUnlock auto_unlock(lock_);
 114     base::PlatformThread::Join(thread_to_join);
 115   }
 116
 117   video_frame_stream_.Stop(callback);
 118 }
 119
 120 void VideoRendererBase::SetPlaybackRate(float playback_rate) {
 121   DCHECK(message_loop_->BelongsToCurrentThread());
 122   base::AutoLock auto_lock(lock_);
 123   playback_rate_ = playback_rate;
 124 }
 125
 126 void VideoRendererBase::Preroll(base::TimeDelta time,
 127                                 const PipelineStatusCB& cb) {
 128   DCHECK(message_loop_->BelongsToCurrentThread());
 129   base::AutoLock auto_lock(lock_);
 130   DCHECK_EQ(state_, kFlushed) << "Must flush prior to prerolling.";
 131   DCHECK(!cb.is_null());
 132   DCHECK(preroll_cb_.is_null());
 133
 134   state_ = kPrerolling;
 135   preroll_cb_ = cb;
 136   preroll_timestamp_ = time;
 137   AttemptRead_Locked();
 138 }
 139
 140 void VideoRendererBase::Initialize(DemuxerStream* stream,
 141                                    const PipelineStatusCB& init_cb,
 142                                    const StatisticsCB& statistics_cb,
 143                                    const TimeCB& max_time_cb,
 144                                    const NaturalSizeChangedCB& size_changed_cb,
 145                                    const base::Closure& ended_cb,
 146                                    const PipelineStatusCB& error_cb,
 147                                    const TimeDeltaCB& get_time_cb,
 148                                    const TimeDeltaCB& get_duration_cb) {
 149   DCHECK(message_loop_->BelongsToCurrentThread());
 150   base::AutoLock auto_lock(lock_);
 151   DCHECK(stream);
 152   DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
 153   DCHECK(!init_cb.is_null());
 154   DCHECK(!statistics_cb.is_null());
 155   DCHECK(!max_time_cb.is_null());
 156   DCHECK(!size_changed_cb.is_null());
 157   DCHECK(!ended_cb.is_null());
 158   DCHECK(!get_time_cb.is_null());
 159   DCHECK(!get_duration_cb.is_null());
 160   DCHECK_EQ(kUninitialized, state_);
 161
 162   weak_this_ = weak_factory_.GetWeakPtr();
 163   init_cb_ = init_cb;
 164   statistics_cb_ = statistics_cb;
 165   max_time_cb_ = max_time_cb;
 166   size_changed_cb_ = size_changed_cb;
 167   ended_cb_ = ended_cb;
 168   error_cb_ = error_cb;
 169   get_time_cb_ = get_time_cb;
 170   get_duration_cb_ = get_duration_cb;
 171   state_ = kInitializing;
 172
 173   video_frame_stream_.Initialize(
 174       stream,
 175       statistics_cb,
 176       base::Bind(&VideoRendererBase::OnVideoFrameStreamInitialized,
 177                  weak_this_));
 178 }
 179
 180 void VideoRendererBase::OnVideoFrameStreamInitialized(bool success,
 181                                                       bool has_alpha) {
 182   DCHECK(message_loop_->BelongsToCurrentThread());
 183   base::AutoLock auto_lock(lock_);
 184
 185   if (state_ == kStopped)
 186     return;
 187
 188   DCHECK_EQ(state_, kInitializing);
 189
 190   if (!success) {
 191     state_ = kUninitialized;
 192     base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
 193     return;
 194   }
 195
 196   // We're all good!  Consider ourselves flushed. (ThreadMain() should never
 197   // see us in the kUninitialized state).
 198   // Since we had an initial Preroll(), we consider ourself flushed, because we
 199   // have not populated any buffers yet.
 200   state_ = kFlushed;
 201
 202   set_opaque_cb_.Run(!has_alpha);
 203   set_opaque_cb_.Reset();
 204
 205   // Create our video thread.
 206   if (!base::PlatformThread::Create(0, this, &thread_)) {
 207     NOTREACHED() << "Video thread creation failed";
 208     state_ = kError;
 209     base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED);
 210     return;
 211   }
 212
 213 #if defined(OS_WIN)
 214   // Bump up our priority so our sleeping is more accurate.
 215   // TODO(scherkus): find out if this is necessary, but it seems to help.
 216   ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL);
 217 #endif  // defined(OS_WIN)
 218   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
 219 }
 220
 221 // PlatformThread::Delegate implementation.
 222 void VideoRendererBase::ThreadMain() {
 223   base::PlatformThread::SetName("CrVideoRenderer");
 224
 225   // The number of milliseconds to idle when we do not have anything to do.
 226   // Nothing special about the value, other than we're being more OS-friendly
 227   // than sleeping for 1 millisecond.
 228   //
 229   // TODO(scherkus): switch to pure event-driven frame timing instead of this
 230   // kIdleTimeDelta business http://crbug.com/106874
 231   const base::TimeDelta kIdleTimeDelta =
 232       base::TimeDelta::FromMilliseconds(10);
 233
 234   for (;;) {
 235     base::AutoLock auto_lock(lock_);
 236
 237     // Thread exit condition.
 238     if (state_ == kStopped)
 239       return;
 240
 241     // Remain idle as long as we're not playing.
 242     if (state_ != kPlaying || playback_rate_ == 0) {
 243       UpdateStatsAndWait_Locked(kIdleTimeDelta);
 244       continue;
 245     }
 246
 247     // Remain idle until we have the next frame ready for rendering.
 248     if (ready_frames_.empty()) {
 249       if (received_end_of_stream_) {
 250         state_ = kEnded;
 251         ended_cb_.Run();
 252
 253         // No need to sleep here as we idle when |state_ != kPlaying|.
 254         continue;
 255       }
 256
 257       UpdateStatsAndWait_Locked(kIdleTimeDelta);
 258       continue;
 259     }
 260
 261     base::TimeDelta remaining_time =
 262         CalculateSleepDuration(ready_frames_.front(), playback_rate_);
 263
 264     // Sleep up to a maximum of our idle time until we're within the time to
 265     // render the next frame.
 266     if (remaining_time.InMicroseconds() > 0) {
 267       remaining_time = std::min(remaining_time, kIdleTimeDelta);
 268       UpdateStatsAndWait_Locked(remaining_time);
 269       continue;
 270     }
 271
 272     // Deadline is defined as the midpoint between this frame and the next
 273     // frame, using the delta between this frame and the previous frame as the
 274     // assumption for frame duration.
 275     //
 276     // TODO(scherkus): An improvement over midpoint might be selecting the
 277     // minimum and/or maximum between the midpoint and some constants. As a
 278     // thought experiment, consider what would be better than the midpoint
 279     // for both the 1fps case and 120fps case.
 280     //
 281     // TODO(scherkus): This can be vastly improved. Use a histogram to measure
 282     // the accuracy of our frame timing code. http://crbug.com/149829
 283     if (drop_frames_ && last_timestamp_ != kNoTimestamp()) {
 284       base::TimeDelta now = get_time_cb_.Run();
 285       base::TimeDelta deadline = ready_frames_.front()->GetTimestamp() +
 286           (ready_frames_.front()->GetTimestamp() - last_timestamp_) / 2;
 287
 288       if (now > deadline) {
 289         DropNextReadyFrame_Locked();
 290         continue;
 291       }
 292     }
 293
 294     // Congratulations! You've made it past the video frame timing gauntlet.
 295     //
 296     // At this point enough time has passed that the next frame that ready for
 297     // rendering.
 298     PaintNextReadyFrame_Locked();
 299   }
 300 }
 301
 302 void VideoRendererBase::PaintNextReadyFrame_Locked() {
 303   lock_.AssertAcquired();
 304
 305   scoped_refptr<VideoFrame> next_frame = ready_frames_.front();
 306   ready_frames_.pop_front();
 307   frames_decoded_++;
 308
 309   last_timestamp_ = next_frame->GetTimestamp();
 310
 311   const gfx::Size& natural_size = next_frame->natural_size();
 312   if (natural_size != last_natural_size_) {
 313     last_natural_size_ = natural_size;
 314     size_changed_cb_.Run(natural_size);
 315   }
 316
 317   paint_cb_.Run(next_frame);
 318
 319   message_loop_->PostTask(FROM_HERE, base::Bind(
 320       &VideoRendererBase::AttemptRead, weak_this_));
 321 }
 322
 323 void VideoRendererBase::DropNextReadyFrame_Locked() {
 324   lock_.AssertAcquired();
 325
 326   last_timestamp_ = ready_frames_.front()->GetTimestamp();
 327   ready_frames_.pop_front();
 328   frames_decoded_++;
 329   frames_dropped_++;
 330
 331   message_loop_->PostTask(FROM_HERE, base::Bind(
 332       &VideoRendererBase::AttemptRead, weak_this_));
 333 }
 334
 335 void VideoRendererBase::FrameReady(VideoFrameStream::Status status,
 336                                    const scoped_refptr<VideoFrame>& frame) {
 337   base::AutoLock auto_lock(lock_);
 338   DCHECK_NE(state_, kUninitialized);
 339   DCHECK_NE(state_, kFlushed);
 340
 341   CHECK(pending_read_);
 342   pending_read_ = false;
 343
 344   if (status == VideoFrameStream::DECODE_ERROR ||
 345       status == VideoFrameStream::DECRYPT_ERROR) {
 346     DCHECK(!frame.get());
 347     PipelineStatus error = PIPELINE_ERROR_DECODE;
 348     if (status == VideoFrameStream::DECRYPT_ERROR)
 349       error = PIPELINE_ERROR_DECRYPT;
 350
 351     if (!preroll_cb_.is_null()) {
 352       base::ResetAndReturn(&preroll_cb_).Run(error);
 353       return;
 354     }
 355
 356     error_cb_.Run(error);
 357     return;
 358   }
 359
 360   // Already-queued VideoFrameStream ReadCB's can fire after various state
 361   // transitions have happened; in that case just drop those frames immediately.
 362   if (state_ == kStopped || state_ == kError || state_ == kFlushing)
 363     return;
 364
 365   if (!frame.get()) {
 366     // Abort preroll early for a NULL frame because we won't get more frames.
 367     // A new preroll will be requested after this one completes so there is no
 368     // point trying to collect more frames.
 369     if (state_ == kPrerolling)
 370       TransitionToPrerolled_Locked();
 371
 372     return;
 373   }
 374
 375   if (frame->IsEndOfStream()) {
 376     DCHECK(!received_end_of_stream_);
 377     received_end_of_stream_ = true;
 378     max_time_cb_.Run(get_duration_cb_.Run());
 379
 380     if (state_ == kPrerolling)
 381       TransitionToPrerolled_Locked();
 382
 383     return;
 384   }
 385
 386   // Maintain the latest frame decoded so the correct frame is displayed after
 387   // prerolling has completed.
 388   if (state_ == kPrerolling && frame->GetTimestamp() <= preroll_timestamp_) {
 389     ready_frames_.clear();
 390   }
 391
 392   AddReadyFrame_Locked(frame);
 393
 394   if (state_ == kPrerolling) {
 395     if (!video_frame_stream_.CanReadWithoutStalling() ||
 396         ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames)) {
 397       TransitionToPrerolled_Locked();
 398     }
 399   }
 400
 401   // Always request more decoded video if we have capacity. This serves two
 402   // purposes:
 403   //   1) Prerolling while paused
 404   //   2) Keeps decoding going if video rendering thread starts falling behind
 405   AttemptRead_Locked();
 406 }
 407
 408 void VideoRendererBase::AddReadyFrame_Locked(
 409     const scoped_refptr<VideoFrame>& frame) {
 410   lock_.AssertAcquired();
 411   DCHECK(!frame->IsEndOfStream());
 412
 413   // Adjust the incoming frame if its rendering stop time is past the duration
 414   // of the video itself. This is typically the last frame of the video and
 415   // occurs if the container specifies a duration that isn't a multiple of the
 416   // frame rate.  Another way for this to happen is for the container to state
 417   // a smaller duration than the largest packet timestamp.
 418   base::TimeDelta duration = get_duration_cb_.Run();
 419   if (frame->GetTimestamp() > duration) {
 420     frame->SetTimestamp(duration);
 421   }
 422
 423   ready_frames_.push_back(frame);
 424   DCHECK_LE(ready_frames_.size(),
 425             static_cast<size_t>(limits::kMaxVideoFrames));
 426
 427   max_time_cb_.Run(frame->GetTimestamp());
 428
 429   // Avoid needlessly waking up |thread_| unless playing.
 430   if (state_ == kPlaying)
 431     frame_available_.Signal();
 432 }
 433
 434 void VideoRendererBase::AttemptRead() {
 435   base::AutoLock auto_lock(lock_);
 436   AttemptRead_Locked();
 437 }
 438
 439 void VideoRendererBase::AttemptRead_Locked() {
 440   DCHECK(message_loop_->BelongsToCurrentThread());
 441   lock_.AssertAcquired();
 442
 443   if (pending_read_ || received_end_of_stream_ ||
 444       ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) {
 445     return;
 446   }
 447
 448   switch (state_) {
 449     case kPaused:
 450     case kPrerolling:
 451     case kPlaying:
 452       pending_read_ = true;
 453       video_frame_stream_.Read(base::Bind(&VideoRendererBase::FrameReady,
 454                                           weak_this_));
 455       return;
 456
 457     case kUninitialized:
 458     case kInitializing:
 459     case kPrerolled:
 460     case kFlushing:
 461     case kFlushed:
 462     case kEnded:
 463     case kStopped:
 464     case kError:
 465       return;
 466   }
 467 }
 468
 469 void VideoRendererBase::OnVideoFrameStreamResetDone() {
 470   base::AutoLock auto_lock(lock_);
 471   if (state_ == kStopped)
 472     return;
 473
 474   DCHECK_EQ(kFlushing, state_);
 475   DCHECK(!pending_read_);
 476   DCHECK(ready_frames_.empty());
 477   DCHECK(!received_end_of_stream_);
 478
 479   state_ = kFlushed;
 480   last_timestamp_ = kNoTimestamp();
 481   base::ResetAndReturn(&flush_cb_).Run();
 482 }
 483
 484 base::TimeDelta VideoRendererBase::CalculateSleepDuration(
 485     const scoped_refptr<VideoFrame>& next_frame,
 486     float playback_rate) {
 487   // Determine the current and next presentation timestamps.
 488   base::TimeDelta now = get_time_cb_.Run();
 489   base::TimeDelta next_pts = next_frame->GetTimestamp();
 490
 491   // Scale our sleep based on the playback rate.
 492   base::TimeDelta sleep = next_pts - now;
 493   return base::TimeDelta::FromMicroseconds(
 494       static_cast<int64>(sleep.InMicroseconds() / playback_rate));
 495 }
 496
 497 void VideoRendererBase::DoStopOrError_Locked() {
 498   lock_.AssertAcquired();
 499   last_timestamp_ = kNoTimestamp();
 500   ready_frames_.clear();
 501 }
 502
 503 void VideoRendererBase::TransitionToPrerolled_Locked() {
 504   lock_.AssertAcquired();
 505   DCHECK_EQ(state_, kPrerolling);
 506
 507   state_ = kPrerolled;
 508
 509   // Because we might remain in the prerolled state for an undetermined amount
 510   // of time (e.g., we seeked while paused), we'll paint the first prerolled
 511   // frame.
 512   if (!ready_frames_.empty())
 513     PaintNextReadyFrame_Locked();
 514
 515   base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK);
 516 }
 517
 518 void VideoRendererBase::UpdateStatsAndWait_Locked(
 519     base::TimeDelta wait_duration) {
 520   lock_.AssertAcquired();
 521   DCHECK_GE(frames_decoded_, 0);
 522   DCHECK_LE(frames_dropped_, frames_decoded_);
 523
 524   if (frames_decoded_) {
 525     PipelineStatistics statistics;
 526     statistics.video_frames_decoded = frames_decoded_;
 527     statistics.video_frames_dropped = frames_dropped_;
 528     statistics_cb_.Run(statistics);
 529
 530     frames_decoded_ = 0;
 531     frames_dropped_ = 0;
 532   }
 533
 534   frame_available_.TimedWait(wait_duration);
 535 }
 536
 537 }  // namespace media