media/filters/video_renderer_base.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/video_renderer_base.h"
   6
   7 #include "base/bind.h"
   8 #include "base/callback.h"
   9 #include "base/callback_helpers.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "base/threading/platform_thread.h"
  12 #include "media/base/buffers.h"
  13 #include "media/base/limits.h"
  14 #include "media/base/pipeline.h"
  15 #include "media/base/video_frame.h"
  16
  17 namespace media {
  18
  19 base::TimeDelta VideoRendererBase::kMaxLastFrameDuration() {
  20   return base::TimeDelta::FromMilliseconds(250);
  21 }
  22
  23 VideoRendererBase::VideoRendererBase(
  24     const scoped_refptr<base::MessageLoopProxy>& message_loop,
  25     ScopedVector<VideoDecoder> decoders,
  26     const SetDecryptorReadyCB& set_decryptor_ready_cb,
  27     const PaintCB& paint_cb,
  28     const SetOpaqueCB& set_opaque_cb,
  29     bool drop_frames)
  30     : message_loop_(message_loop),
  31       weak_factory_(this),
  32       video_frame_stream_(
  33           message_loop, decoders.Pass(), set_decryptor_ready_cb),
  34       received_end_of_stream_(false),
  35       frame_available_(&lock_),
  36       state_(kUninitialized),
  37       thread_(),
  38       pending_read_(false),
  39       drop_frames_(drop_frames),
  40       playback_rate_(0),
  41       paint_cb_(paint_cb),
  42       set_opaque_cb_(set_opaque_cb),
  43       last_timestamp_(kNoTimestamp()) {
  44   DCHECK(!paint_cb_.is_null());
  45 }
  46
  47 VideoRendererBase::~VideoRendererBase() {
  48   base::AutoLock auto_lock(lock_);
  49   CHECK(state_ == kStopped || state_ == kUninitialized) << state_;
  50   CHECK(thread_.is_null());
  51 }
  52
  53 void VideoRendererBase::Play(const base::Closure& callback) {
  54   DCHECK(message_loop_->BelongsToCurrentThread());
  55   base::AutoLock auto_lock(lock_);
  56   DCHECK_EQ(kPrerolled, state_);
  57   state_ = kPlaying;
  58   callback.Run();
  59 }
  60
  61 void VideoRendererBase::Pause(const base::Closure& callback) {
  62   DCHECK(message_loop_->BelongsToCurrentThread());
  63   base::AutoLock auto_lock(lock_);
  64   DCHECK(state_ != kUninitialized || state_ == kError);
  65   state_ = kPaused;
  66   callback.Run();
  67 }
  68
  69 void VideoRendererBase::Flush(const base::Closure& callback) {
  70   DCHECK(message_loop_->BelongsToCurrentThread());
  71   base::AutoLock auto_lock(lock_);
  72   DCHECK_EQ(state_, kPaused);
  73   flush_cb_ = callback;
  74   state_ = kFlushing;
  75
  76   // This is necessary if the |video_frame_stream_| has already seen an end of
  77   // stream and needs to drain it before flushing it.
  78   ready_frames_.clear();
  79   received_end_of_stream_ = false;
  80   video_frame_stream_.Reset(base::Bind(
  81       &VideoRendererBase::OnVideoFrameStreamResetDone, weak_this_));
  82 }
  83
  84 void VideoRendererBase::Stop(const base::Closure& callback) {
  85   DCHECK(message_loop_->BelongsToCurrentThread());
  86   base::AutoLock auto_lock(lock_);
  87   if (state_ == kUninitialized || state_ == kStopped) {
  88     callback.Run();
  89     return;
  90   }
  91
  92   // TODO(scherkus): Consider invalidating |weak_factory_| and replacing
  93   // task-running guards that check |state_| with DCHECK().
  94
  95   state_ = kStopped;
  96
  97   statistics_cb_.Reset();
  98   max_time_cb_.Reset();
  99   DoStopOrError_Locked();
 100
 101   // Clean up our thread if present.
 102   base::PlatformThreadHandle thread_to_join = base::PlatformThreadHandle();
 103   if (!thread_.is_null()) {
 104     // Signal the thread since it's possible to get stopped with the video
 105     // thread waiting for a read to complete.
 106     frame_available_.Signal();
 107     std::swap(thread_, thread_to_join);
 108   }
 109
 110   if (!thread_to_join.is_null()) {
 111     base::AutoUnlock auto_unlock(lock_);
 112     base::PlatformThread::Join(thread_to_join);
 113   }
 114
 115   video_frame_stream_.Stop(callback);
 116 }
 117
 118 void VideoRendererBase::SetPlaybackRate(float playback_rate) {
 119   DCHECK(message_loop_->BelongsToCurrentThread());
 120   base::AutoLock auto_lock(lock_);
 121   playback_rate_ = playback_rate;
 122 }
 123
 124 void VideoRendererBase::Preroll(base::TimeDelta time,
 125                                 const PipelineStatusCB& cb) {
 126   DCHECK(message_loop_->BelongsToCurrentThread());
 127   base::AutoLock auto_lock(lock_);
 128   DCHECK_EQ(state_, kFlushed) << "Must flush prior to prerolling.";
 129   DCHECK(!cb.is_null());
 130   DCHECK(preroll_cb_.is_null());
 131
 132   state_ = kPrerolling;
 133   preroll_cb_ = cb;
 134   preroll_timestamp_ = time;
 135   AttemptRead_Locked();
 136 }
 137
 138 void VideoRendererBase::Initialize(DemuxerStream* stream,
 139                                    const PipelineStatusCB& init_cb,
 140                                    const StatisticsCB& statistics_cb,
 141                                    const TimeCB& max_time_cb,
 142                                    const NaturalSizeChangedCB& size_changed_cb,
 143                                    const base::Closure& ended_cb,
 144                                    const PipelineStatusCB& error_cb,
 145                                    const TimeDeltaCB& get_time_cb,
 146                                    const TimeDeltaCB& get_duration_cb) {
 147   DCHECK(message_loop_->BelongsToCurrentThread());
 148   base::AutoLock auto_lock(lock_);
 149   DCHECK(stream);
 150   DCHECK_EQ(stream->type(), DemuxerStream::VIDEO);
 151   DCHECK(!init_cb.is_null());
 152   DCHECK(!statistics_cb.is_null());
 153   DCHECK(!max_time_cb.is_null());
 154   DCHECK(!size_changed_cb.is_null());
 155   DCHECK(!ended_cb.is_null());
 156   DCHECK(!get_time_cb.is_null());
 157   DCHECK(!get_duration_cb.is_null());
 158   DCHECK_EQ(kUninitialized, state_);
 159
 160   weak_this_ = weak_factory_.GetWeakPtr();
 161   init_cb_ = init_cb;
 162   statistics_cb_ = statistics_cb;
 163   max_time_cb_ = max_time_cb;
 164   size_changed_cb_ = size_changed_cb;
 165   ended_cb_ = ended_cb;
 166   error_cb_ = error_cb;
 167   get_time_cb_ = get_time_cb;
 168   get_duration_cb_ = get_duration_cb;
 169   state_ = kInitializing;
 170
 171   video_frame_stream_.Initialize(
 172       stream,
 173       statistics_cb,
 174       base::Bind(&VideoRendererBase::OnVideoFrameStreamInitialized,
 175                  weak_this_));
 176 }
 177
 178 void VideoRendererBase::OnVideoFrameStreamInitialized(bool success,
 179                                                       bool has_alpha) {
 180   DCHECK(message_loop_->BelongsToCurrentThread());
 181   base::AutoLock auto_lock(lock_);
 182
 183   if (state_ == kStopped)
 184     return;
 185
 186   DCHECK_EQ(state_, kInitializing);
 187
 188   if (!success) {
 189     state_ = kUninitialized;
 190     base::ResetAndReturn(&init_cb_).Run(DECODER_ERROR_NOT_SUPPORTED);
 191     return;
 192   }
 193
 194   // We're all good!  Consider ourselves flushed. (ThreadMain() should never
 195   // see us in the kUninitialized state).
 196   // Since we had an initial Preroll(), we consider ourself flushed, because we
 197   // have not populated any buffers yet.
 198   state_ = kFlushed;
 199
 200   set_opaque_cb_.Run(!has_alpha);
 201   set_opaque_cb_.Reset();
 202
 203   // Create our video thread.
 204   if (!base::PlatformThread::Create(0, this, &thread_)) {
 205     NOTREACHED() << "Video thread creation failed";
 206     state_ = kError;
 207     base::ResetAndReturn(&init_cb_).Run(PIPELINE_ERROR_INITIALIZATION_FAILED);
 208     return;
 209   }
 210
 211 #if defined(OS_WIN)
 212   // Bump up our priority so our sleeping is more accurate.
 213   // TODO(scherkus): find out if this is necessary, but it seems to help.
 214   ::SetThreadPriority(thread_.platform_handle(), THREAD_PRIORITY_ABOVE_NORMAL);
 215 #endif  // defined(OS_WIN)
 216   base::ResetAndReturn(&init_cb_).Run(PIPELINE_OK);
 217 }
 218
 219 // PlatformThread::Delegate implementation.
 220 void VideoRendererBase::ThreadMain() {
 221   base::PlatformThread::SetName("CrVideoRenderer");
 222
 223   // The number of milliseconds to idle when we do not have anything to do.
 224   // Nothing special about the value, other than we're being more OS-friendly
 225   // than sleeping for 1 millisecond.
 226   //
 227   // TODO(scherkus): switch to pure event-driven frame timing instead of this
 228   // kIdleTimeDelta business http://crbug.com/106874
 229   const base::TimeDelta kIdleTimeDelta =
 230       base::TimeDelta::FromMilliseconds(10);
 231
 232   for (;;) {
 233     base::AutoLock auto_lock(lock_);
 234
 235     // Thread exit condition.
 236     if (state_ == kStopped)
 237       return;
 238
 239     // Remain idle as long as we're not playing.
 240     if (state_ != kPlaying || playback_rate_ == 0) {
 241       frame_available_.TimedWait(kIdleTimeDelta);
 242       continue;
 243     }
 244
 245     // Remain idle until we have the next frame ready for rendering.
 246     if (ready_frames_.empty()) {
 247       if (received_end_of_stream_) {
 248         state_ = kEnded;
 249         ended_cb_.Run();
 250
 251         // No need to sleep here as we idle when |state_ != kPlaying|.
 252         continue;
 253       }
 254
 255       frame_available_.TimedWait(kIdleTimeDelta);
 256       continue;
 257     }
 258
 259     base::TimeDelta remaining_time =
 260         CalculateSleepDuration(ready_frames_.front(), playback_rate_);
 261
 262     // Sleep up to a maximum of our idle time until we're within the time to
 263     // render the next frame.
 264     if (remaining_time.InMicroseconds() > 0) {
 265       remaining_time = std::min(remaining_time, kIdleTimeDelta);
 266       frame_available_.TimedWait(remaining_time);
 267       continue;
 268     }
 269
 270     // Deadline is defined as the midpoint between this frame and the next
 271     // frame, using the delta between this frame and the previous frame as the
 272     // assumption for frame duration.
 273     //
 274     // TODO(scherkus): An improvement over midpoint might be selecting the
 275     // minimum and/or maximum between the midpoint and some constants. As a
 276     // thought experiment, consider what would be better than the midpoint
 277     // for both the 1fps case and 120fps case.
 278     //
 279     // TODO(scherkus): This can be vastly improved. Use a histogram to measure
 280     // the accuracy of our frame timing code. http://crbug.com/149829
 281     if (drop_frames_ && last_timestamp_ != kNoTimestamp()) {
 282       base::TimeDelta now = get_time_cb_.Run();
 283       base::TimeDelta deadline = ready_frames_.front()->GetTimestamp() +
 284           (ready_frames_.front()->GetTimestamp() - last_timestamp_) / 2;
 285
 286       if (now > deadline) {
 287         DropNextReadyFrame_Locked();
 288         continue;
 289       }
 290     }
 291
 292     // Congratulations! You've made it past the video frame timing gauntlet.
 293     //
 294     // At this point enough time has passed that the next frame that ready for
 295     // rendering.
 296     PaintNextReadyFrame_Locked();
 297   }
 298 }
 299
 300 void VideoRendererBase::PaintNextReadyFrame_Locked() {
 301   lock_.AssertAcquired();
 302
 303   scoped_refptr<VideoFrame> next_frame = ready_frames_.front();
 304   ready_frames_.pop_front();
 305
 306   last_timestamp_ = next_frame->GetTimestamp();
 307
 308   const gfx::Size& natural_size = next_frame->natural_size();
 309   if (natural_size != last_natural_size_) {
 310     last_natural_size_ = natural_size;
 311     size_changed_cb_.Run(natural_size);
 312   }
 313
 314   paint_cb_.Run(next_frame);
 315
 316   message_loop_->PostTask(FROM_HERE, base::Bind(
 317       &VideoRendererBase::AttemptRead, weak_this_));
 318 }
 319
 320 void VideoRendererBase::DropNextReadyFrame_Locked() {
 321   lock_.AssertAcquired();
 322
 323   last_timestamp_ = ready_frames_.front()->GetTimestamp();
 324   ready_frames_.pop_front();
 325
 326   PipelineStatistics statistics;
 327   statistics.video_frames_dropped = 1;
 328   statistics_cb_.Run(statistics);
 329
 330   message_loop_->PostTask(FROM_HERE, base::Bind(
 331       &VideoRendererBase::AttemptRead, weak_this_));
 332 }
 333
 334 void VideoRendererBase::FrameReady(VideoFrameStream::Status status,
 335                                    const scoped_refptr<VideoFrame>& frame) {
 336   base::AutoLock auto_lock(lock_);
 337   DCHECK_NE(state_, kUninitialized);
 338   DCHECK_NE(state_, kFlushed);
 339
 340   CHECK(pending_read_);
 341   pending_read_ = false;
 342
 343   if (status == VideoFrameStream::DECODE_ERROR ||
 344       status == VideoFrameStream::DECRYPT_ERROR) {
 345     DCHECK(!frame.get());
 346     PipelineStatus error = PIPELINE_ERROR_DECODE;
 347     if (status == VideoFrameStream::DECRYPT_ERROR)
 348       error = PIPELINE_ERROR_DECRYPT;
 349
 350     if (!preroll_cb_.is_null()) {
 351       base::ResetAndReturn(&preroll_cb_).Run(error);
 352       return;
 353     }
 354
 355     error_cb_.Run(error);
 356     return;
 357   }
 358
 359   // Already-queued VideoFrameStream ReadCB's can fire after various state
 360   // transitions have happened; in that case just drop those frames immediately.
 361   if (state_ == kStopped || state_ == kError || state_ == kFlushing)
 362     return;
 363
 364   if (!frame.get()) {
 365     // Abort preroll early for a NULL frame because we won't get more frames.
 366     // A new preroll will be requested after this one completes so there is no
 367     // point trying to collect more frames.
 368     if (state_ == kPrerolling)
 369       TransitionToPrerolled_Locked();
 370
 371     return;
 372   }
 373
 374   if (frame->IsEndOfStream()) {
 375     DCHECK(!received_end_of_stream_);
 376     received_end_of_stream_ = true;
 377     max_time_cb_.Run(get_duration_cb_.Run());
 378
 379     if (state_ == kPrerolling)
 380       TransitionToPrerolled_Locked();
 381
 382     return;
 383   }
 384
 385   // Maintain the latest frame decoded so the correct frame is displayed after
 386   // prerolling has completed.
 387   if (state_ == kPrerolling && frame->GetTimestamp() <= preroll_timestamp_) {
 388     ready_frames_.clear();
 389   }
 390
 391   AddReadyFrame_Locked(frame);
 392
 393   if (state_ == kPrerolling) {
 394     if (!video_frame_stream_.CanReadWithoutStalling() ||
 395         ready_frames_.size() >= static_cast<size_t>(limits::kMaxVideoFrames)) {
 396       TransitionToPrerolled_Locked();
 397     }
 398   } else {
 399     // We only count frames decoded during normal playback.
 400     PipelineStatistics statistics;
 401     statistics.video_frames_decoded = 1;
 402     statistics_cb_.Run(statistics);
 403   }
 404
 405   // Always request more decoded video if we have capacity. This serves two
 406   // purposes:
 407   //   1) Prerolling while paused
 408   //   2) Keeps decoding going if video rendering thread starts falling behind
 409   AttemptRead_Locked();
 410 }
 411
 412 void VideoRendererBase::AddReadyFrame_Locked(
 413     const scoped_refptr<VideoFrame>& frame) {
 414   lock_.AssertAcquired();
 415   DCHECK(!frame->IsEndOfStream());
 416
 417   // Adjust the incoming frame if its rendering stop time is past the duration
 418   // of the video itself. This is typically the last frame of the video and
 419   // occurs if the container specifies a duration that isn't a multiple of the
 420   // frame rate.  Another way for this to happen is for the container to state
 421   // a smaller duration than the largest packet timestamp.
 422   base::TimeDelta duration = get_duration_cb_.Run();
 423   if (frame->GetTimestamp() > duration) {
 424     frame->SetTimestamp(duration);
 425   }
 426
 427   ready_frames_.push_back(frame);
 428   DCHECK_LE(ready_frames_.size(),
 429             static_cast<size_t>(limits::kMaxVideoFrames));
 430
 431   max_time_cb_.Run(frame->GetTimestamp());
 432
 433   // Avoid needlessly waking up |thread_| unless playing.
 434   if (state_ == kPlaying)
 435     frame_available_.Signal();
 436 }
 437
 438 void VideoRendererBase::AttemptRead() {
 439   base::AutoLock auto_lock(lock_);
 440   AttemptRead_Locked();
 441 }
 442
 443 void VideoRendererBase::AttemptRead_Locked() {
 444   DCHECK(message_loop_->BelongsToCurrentThread());
 445   lock_.AssertAcquired();
 446
 447   if (pending_read_ || received_end_of_stream_ ||
 448       ready_frames_.size() == static_cast<size_t>(limits::kMaxVideoFrames)) {
 449     return;
 450   }
 451
 452   switch (state_) {
 453     case kPaused:
 454     case kPrerolling:
 455     case kPlaying:
 456       pending_read_ = true;
 457       video_frame_stream_.Read(base::Bind(&VideoRendererBase::FrameReady,
 458                                           weak_this_));
 459       return;
 460
 461     case kUninitialized:
 462     case kInitializing:
 463     case kPrerolled:
 464     case kFlushing:
 465     case kFlushed:
 466     case kEnded:
 467     case kStopped:
 468     case kError:
 469       return;
 470   }
 471 }
 472
 473 void VideoRendererBase::OnVideoFrameStreamResetDone() {
 474   base::AutoLock auto_lock(lock_);
 475   if (state_ == kStopped)
 476     return;
 477
 478   DCHECK_EQ(kFlushing, state_);
 479   DCHECK(!pending_read_);
 480   DCHECK(ready_frames_.empty());
 481   DCHECK(!received_end_of_stream_);
 482
 483   state_ = kFlushed;
 484   last_timestamp_ = kNoTimestamp();
 485   base::ResetAndReturn(&flush_cb_).Run();
 486 }
 487
 488 base::TimeDelta VideoRendererBase::CalculateSleepDuration(
 489     const scoped_refptr<VideoFrame>& next_frame,
 490     float playback_rate) {
 491   // Determine the current and next presentation timestamps.
 492   base::TimeDelta now = get_time_cb_.Run();
 493   base::TimeDelta next_pts = next_frame->GetTimestamp();
 494
 495   // Scale our sleep based on the playback rate.
 496   base::TimeDelta sleep = next_pts - now;
 497   return base::TimeDelta::FromMicroseconds(
 498       static_cast<int64>(sleep.InMicroseconds() / playback_rate));
 499 }
 500
 501 void VideoRendererBase::DoStopOrError_Locked() {
 502   lock_.AssertAcquired();
 503   last_timestamp_ = kNoTimestamp();
 504   ready_frames_.clear();
 505 }
 506
 507 void VideoRendererBase::TransitionToPrerolled_Locked() {
 508   lock_.AssertAcquired();
 509   DCHECK_EQ(state_, kPrerolling);
 510
 511   state_ = kPrerolled;
 512
 513   // Because we might remain in the prerolled state for an undetermined amount
 514   // of time (e.g., we seeked while paused), we'll paint the first prerolled
 515   // frame.
 516   if (!ready_frames_.empty())
 517     PaintNextReadyFrame_Locked();
 518
 519   base::ResetAndReturn(&preroll_cb_).Run(PIPELINE_OK);
 520 }
 521
 522 }  // namespace media