content/renderer/media/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/command_line.h"
  13 #include "base/location.h"
  14 #include "base/metrics/histogram.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/trace_event/trace_event.h"
  17 #include "content/public/common/content_switches.h"
  18 #include "media/base/bind_to_current_loop.h"
  19 #include "media/base/video_util.h"
  20
  21 namespace content {
  22
  23 namespace {
  24
  25 // Amount of frame intervals to wait before considering the source as muted, for
  26 // the first frame and under normal conditions, respectively. First frame might
  27 // take longer to arrive due to source startup.
  28 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  29 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  30
  31 // Min delta time between two frames allowed without being dropped if a max
  32 // frame rate is specified.
  33 const double kMinTimeInMsBetweenFrames = 5;
  34 // If the delta between two frames is bigger than this, we will consider it to
  35 // be invalid and reset the fps calculation.
  36 const double kMaxTimeInMsBetweenFrames = 1000;
  37
  38 // Empty method used for keeping a reference to the original media::VideoFrame
  39 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
  40 // The reference to |frame| is kept in the closure that calls this method.
  41 void ReleaseOriginalFrame(const scoped_refptr<media::VideoFrame>& frame) {
  42 }
  43
  44 void ResetCallbackOnMainRenderThread(
  45     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
  46   // |callback| will be deleted when this exits.
  47 }
  48
  49 }  // anonymous namespace
  50
  51 // VideoFrameResolutionAdapter is created on and lives on the IO-thread. It does
  52 // the resolution adaptation and delivers frames to all registered tracks on the
  53 // IO-thread. All method calls must be on the IO-thread.
  54 class VideoTrackAdapter::VideoFrameResolutionAdapter
  55     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
  56  public:
  57   // Setting |max_frame_rate| to 0.0, means that no frame rate limitation
  58   // will be done.
  59   VideoFrameResolutionAdapter(
  60       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
  61       const gfx::Size& max_size,
  62       double min_aspect_ratio,
  63       double max_aspect_ratio,
  64       double max_frame_rate);
  65
  66   // Add |callback| to receive video frames on the IO-thread.
  67   // |callback| will however be released on the main render thread.
  68   void AddCallback(const MediaStreamVideoTrack* track,
  69                    const VideoCaptureDeliverFrameCB& callback);
  70
  71   // Removes |callback| associated with |track| from receiving video frames if
  72   // |track| has been added. It is ok to call RemoveCallback even if the |track|
  73   // has not been added. The |callback| is released on the main render thread.
  74   void RemoveCallback(const MediaStreamVideoTrack* track);
  75
  76   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  77                     const base::TimeTicks& estimated_capture_time);
  78
  79   // Returns true if all arguments match with the output of this adapter.
  80   bool ConstraintsMatch(const gfx::Size& max_size,
  81                         double min_aspect_ratio,
  82                         double max_aspect_ratio,
  83                         double max_frame_rate) const;
  84
  85   bool IsEmpty() const;
  86
  87  private:
  88   virtual ~VideoFrameResolutionAdapter();
  89   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
  90
  91   void DoDeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  92                       const base::TimeTicks& estimated_capture_time);
  93
  94   // Returns |true| if the input frame rate is higher that the requested max
  95   // frame rate and |frame| should be dropped.
  96   bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame,
  97                       float source_frame_rate);
  98
  99   // Bound to the IO-thread.
 100   base::ThreadChecker io_thread_checker_;
 101
 102   // The task runner where we will release VideoCaptureDeliverFrameCB
 103   // registered in AddCallback.
 104   const scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 105
 106   const gfx::Size max_frame_size_;
 107   const double min_aspect_ratio_;
 108   const double max_aspect_ratio_;
 109
 110   double frame_rate_;
 111   base::TimeDelta last_time_stamp_;
 112   double max_frame_rate_;
 113   double keep_frame_counter_;
 114
 115   typedef std::pair<const MediaStreamVideoTrack*, VideoCaptureDeliverFrameCB>
 116       VideoIdCallbackPair;
 117   std::vector<VideoIdCallbackPair> callbacks_;
 118
 119   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 120 };
 121
 122 VideoTrackAdapter::
 123 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 124     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
 125     const gfx::Size& max_size,
 126     double min_aspect_ratio,
 127     double max_aspect_ratio,
 128     double max_frame_rate)
 129     : renderer_task_runner_(render_message_loop),
 130       max_frame_size_(max_size),
 131       min_aspect_ratio_(min_aspect_ratio),
 132       max_aspect_ratio_(max_aspect_ratio),
 133       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
 134       last_time_stamp_(base::TimeDelta::Max()),
 135       max_frame_rate_(max_frame_rate),
 136       keep_frame_counter_(0.0) {
 137   DCHECK(renderer_task_runner_.get());
 138   DCHECK(io_thread_checker_.CalledOnValidThread());
 139   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
 140   CHECK_NE(0, max_aspect_ratio_);
 141
 142   const std::string max_fps_str =
 143       base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 144           switches::kWebRtcMaxCaptureFramerate);
 145   if (!max_fps_str.empty()) {
 146     double value;
 147     if (base::StringToDouble(max_fps_str, &value) && value >= 0.0) {
 148       DVLOG(1) << "Overriding max frame rate.  Was=" << max_frame_rate
 149               << ", Now=" << value;
 150       max_frame_rate_ = value;
 151     } else {
 152       DLOG(ERROR) << "Unable to set max fps to " << max_fps_str;
 153     }
 154   }
 155
 156   DVLOG(3) << "VideoFrameResolutionAdapter("
 157           << "{ max_width =" << max_frame_size_.width() << "}, "
 158           << "{ max_height =" << max_frame_size_.height() << "}, "
 159           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
 160           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}"
 161           << "{ max_frame_rate_ =" << max_frame_rate_ << "}) ";
 162 }
 163
 164 VideoTrackAdapter::
 165 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 166   DCHECK(io_thread_checker_.CalledOnValidThread());
 167   DCHECK(callbacks_.empty());
 168 }
 169
 170 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
 171     const MediaStreamVideoTrack* track,
 172     const VideoCaptureDeliverFrameCB& callback) {
 173   DCHECK(io_thread_checker_.CalledOnValidThread());
 174   callbacks_.push_back(std::make_pair(track, callback));
 175 }
 176
 177 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
 178     const MediaStreamVideoTrack* track) {
 179   DCHECK(io_thread_checker_.CalledOnValidThread());
 180   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
 181   for (; it != callbacks_.end(); ++it) {
 182     if (it->first == track) {
 183       // Make sure the VideoCaptureDeliverFrameCB is released on the main
 184       // render thread since it was added on the main render thread in
 185       // VideoTrackAdapter::AddTrack.
 186       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
 187           new VideoCaptureDeliverFrameCB(it->second));
 188       callbacks_.erase(it);
 189       renderer_task_runner_->PostTask(
 190           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
 191                                 base::Passed(&callback)));
 192
 193       return;
 194     }
 195   }
 196 }
 197
 198 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 199     const scoped_refptr<media::VideoFrame>& frame,
 200     const base::TimeTicks& estimated_capture_time) {
 201   DCHECK(io_thread_checker_.CalledOnValidThread());
 202
 203   double frame_rate;
 204   if (!frame->metadata()->GetDouble(media::VideoFrameMetadata::FRAME_RATE,
 205                                     &frame_rate)) {
 206     frame_rate = MediaStreamVideoSource::kUnknownFrameRate;
 207   }
 208
 209   if (MaybeDropFrame(frame, frame_rate))
 210     return;
 211
 212   // TODO(perkj): Allow cropping / scaling of textures once
 213   // http://crbug/362521 is fixed.
 214   if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
 215     DoDeliverFrame(frame, estimated_capture_time);
 216     return;
 217   }
 218   scoped_refptr<media::VideoFrame> video_frame(frame);
 219   double input_ratio =
 220       static_cast<double>(frame->natural_size().width()) /
 221       frame->natural_size().height();
 222
 223   // If |frame| has larger width or height than requested, or the aspect ratio
 224   // does not match the requested, we want to create a wrapped version of this
 225   // frame with a size that fulfills the constraints.
 226   if (frame->natural_size().width() > max_frame_size_.width() ||
 227       frame->natural_size().height() > max_frame_size_.height() ||
 228       input_ratio > max_aspect_ratio_ ||
 229       input_ratio < min_aspect_ratio_) {
 230     int desired_width = std::min(max_frame_size_.width(),
 231                                  frame->natural_size().width());
 232     int desired_height = std::min(max_frame_size_.height(),
 233                                   frame->natural_size().height());
 234
 235     const double resulting_ratio =
 236         static_cast<double>(desired_width) / desired_height;
 237     // Make sure |min_aspect_ratio_| < |requested_ratio| < |max_aspect_ratio_|.
 238     const double requested_ratio = std::max(
 239         std::min(resulting_ratio, max_aspect_ratio_), min_aspect_ratio_);
 240
 241     if (resulting_ratio < requested_ratio) {
 242       desired_height = static_cast<int>((desired_height * resulting_ratio) /
 243                                         requested_ratio);
 244       // Make sure we scale to an even height to avoid rounding errors
 245       desired_height = (desired_height + 1) & ~1;
 246     } else if (resulting_ratio > requested_ratio) {
 247       desired_width = static_cast<int>((desired_width * requested_ratio) /
 248                                        resulting_ratio);
 249       // Make sure we scale to an even width to avoid rounding errors.
 250       desired_width = (desired_width + 1) & ~1;
 251     }
 252
 253     const gfx::Size desired_size(desired_width, desired_height);
 254
 255     // Get the largest centered rectangle with the same aspect ratio of
 256     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
 257     // This will be the rect we need to crop the original frame to.
 258     // From this rect, the original frame can be scaled down to |desired_size|.
 259     const gfx::Rect region_in_frame =
 260         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 261
 262     video_frame = media::VideoFrame::WrapVideoFrame(
 263         frame,
 264         region_in_frame,
 265         desired_size,
 266         base::Bind(&ReleaseOriginalFrame, frame));
 267
 268     DVLOG(3) << "desired size  " << desired_size.ToString()
 269              << " output natural size "
 270              << video_frame->natural_size().ToString()
 271              << " output visible rect  "
 272              << video_frame->visible_rect().ToString();
 273   }
 274   DoDeliverFrame(video_frame, estimated_capture_time);
 275 }
 276
 277 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
 278     const gfx::Size& max_size,
 279     double min_aspect_ratio,
 280     double max_aspect_ratio,
 281     double max_frame_rate) const {
 282   DCHECK(io_thread_checker_.CalledOnValidThread());
 283   return max_frame_size_ == max_size &&
 284       min_aspect_ratio_ == min_aspect_ratio &&
 285       max_aspect_ratio_ == max_aspect_ratio &&
 286       max_frame_rate_ == max_frame_rate;
 287 }
 288
 289 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 290   DCHECK(io_thread_checker_.CalledOnValidThread());
 291   return callbacks_.empty();
 292 }
 293
 294 void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame(
 295     const scoped_refptr<media::VideoFrame>& frame,
 296     const base::TimeTicks& estimated_capture_time) {
 297   DCHECK(io_thread_checker_.CalledOnValidThread());
 298   for (const auto& callback : callbacks_)
 299     callback.second.Run(frame, estimated_capture_time);
 300 }
 301
 302 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 303     const scoped_refptr<media::VideoFrame>& frame,
 304     float source_frame_rate) {
 305   DCHECK(io_thread_checker_.CalledOnValidThread());
 306
 307   // Do not drop frames if max frame rate hasn't been specified or the source
 308   // frame rate is known and is lower than max.
 309   if (max_frame_rate_ == 0.0f ||
 310       (source_frame_rate > 0 && source_frame_rate <= max_frame_rate_)) {
 311     return false;
 312   }
 313
 314   const double delta_ms =
 315       (frame->timestamp() - last_time_stamp_).InMillisecondsF();
 316
 317   // Check if the time since the last frame is completely off.
 318   if (delta_ms < 0 || delta_ms > kMaxTimeInMsBetweenFrames) {
 319     // Reset |last_time_stamp_| and fps calculation.
 320     last_time_stamp_ = frame->timestamp();
 321     frame_rate_ = MediaStreamVideoSource::kDefaultFrameRate;
 322     keep_frame_counter_ = 0.0;
 323     return false;
 324   }
 325
 326   if (delta_ms < kMinTimeInMsBetweenFrames) {
 327     // We have seen video frames being delivered from camera devices back to
 328     // back. The simple AR filter for frame rate calculation is too short to
 329     // handle that. http://crbug/394315
 330     // TODO(perkj): Can we come up with a way to fix the times stamps and the
 331     // timing when frames are delivered so all frames can be used?
 332     // The time stamps are generated by Chrome and not the actual device.
 333     // Most likely the back to back problem is caused by software and not the
 334     // actual camera.
 335     DVLOG(3) << "Drop frame since delta time since previous frame is "
 336              << delta_ms << "ms.";
 337     return true;
 338   }
 339   last_time_stamp_ = frame->timestamp();
 340   // Calculate the frame rate using a simple AR filter.
 341   // Use a simple filter with 0.1 weight of the current sample.
 342   frame_rate_ = 100 / delta_ms + 0.9 * frame_rate_;
 343
 344   // Prefer to not drop frames.
 345   if (max_frame_rate_ + 0.5f > frame_rate_)
 346     return false;  // Keep this frame.
 347
 348   // The input frame rate is higher than requested.
 349   // Decide if we should keep this frame or drop it.
 350   keep_frame_counter_ += max_frame_rate_ / frame_rate_;
 351   if (keep_frame_counter_ >= 1) {
 352     keep_frame_counter_ -= 1;
 353     // Keep the frame.
 354     return false;
 355   }
 356   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
 357   return true;
 358 }
 359
 360 VideoTrackAdapter::VideoTrackAdapter(
 361     const scoped_refptr<base::MessageLoopProxy>& io_message_loop)
 362     : io_message_loop_(io_message_loop),
 363       renderer_task_runner_(base::MessageLoopProxy::current()),
 364       monitoring_frame_rate_(false),
 365       muted_state_(false),
 366       frame_counter_(0),
 367       source_frame_rate_(0.0f) {
 368   DCHECK(io_message_loop_.get());
 369 }
 370
 371 VideoTrackAdapter::~VideoTrackAdapter() {
 372   DCHECK(adapters_.empty());
 373 }
 374
 375 void VideoTrackAdapter::AddTrack(const MediaStreamVideoTrack* track,
 376                                  VideoCaptureDeliverFrameCB frame_callback,
 377                                  int max_width,
 378                                  int max_height,
 379                                  double min_aspect_ratio,
 380                                  double max_aspect_ratio,
 381                                  double max_frame_rate) {
 382   DCHECK(thread_checker_.CalledOnValidThread());
 383
 384   io_message_loop_->PostTask(
 385       FROM_HERE,
 386       base::Bind(&VideoTrackAdapter::AddTrackOnIO,
 387                  this, track, frame_callback, gfx::Size(max_width, max_height),
 388                  min_aspect_ratio, max_aspect_ratio, max_frame_rate));
 389 }
 390
 391 void VideoTrackAdapter::AddTrackOnIO(const MediaStreamVideoTrack* track,
 392                                      VideoCaptureDeliverFrameCB frame_callback,
 393                                      const gfx::Size& max_frame_size,
 394                                      double min_aspect_ratio,
 395                                      double max_aspect_ratio,
 396                                      double max_frame_rate) {
 397   DCHECK(io_message_loop_->BelongsToCurrentThread());
 398   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 399   for (const auto& frame_adapter : adapters_) {
 400     if (frame_adapter->ConstraintsMatch(max_frame_size, min_aspect_ratio,
 401                                         max_aspect_ratio, max_frame_rate)) {
 402       adapter = frame_adapter.get();
 403       break;
 404     }
 405   }
 406   if (!adapter.get()) {
 407     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
 408                                               max_frame_size,
 409                                               min_aspect_ratio,
 410                                               max_aspect_ratio,
 411                                               max_frame_rate);
 412     adapters_.push_back(adapter);
 413   }
 414
 415   adapter->AddCallback(track, frame_callback);
 416 }
 417
 418 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 419   DCHECK(thread_checker_.CalledOnValidThread());
 420   io_message_loop_->PostTask(
 421       FROM_HERE,
 422       base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
 423 }
 424
 425 void VideoTrackAdapter::StartFrameMonitoring(
 426     double source_frame_rate,
 427     const OnMutedCallback& on_muted_callback) {
 428   DCHECK(thread_checker_.CalledOnValidThread());
 429
 430   VideoTrackAdapter::OnMutedCallback bound_on_muted_callback =
 431       media::BindToCurrentLoop(on_muted_callback);
 432
 433   io_message_loop_->PostTask(
 434       FROM_HERE,
 435       base::Bind(&VideoTrackAdapter::StartFrameMonitoringOnIO,
 436                  this, bound_on_muted_callback, source_frame_rate));
 437 }
 438
 439 void VideoTrackAdapter::StopFrameMonitoring() {
 440   DCHECK(thread_checker_.CalledOnValidThread());
 441   io_message_loop_->PostTask(
 442       FROM_HERE,
 443       base::Bind(&VideoTrackAdapter::StopFrameMonitoringOnIO, this));
 444 }
 445
 446 void VideoTrackAdapter::StartFrameMonitoringOnIO(
 447     const OnMutedCallback& on_muted_callback,
 448     double source_frame_rate) {
 449   DCHECK(io_message_loop_->BelongsToCurrentThread());
 450   DCHECK(!monitoring_frame_rate_);
 451
 452   monitoring_frame_rate_ = true;
 453
 454   // If the source does not know the frame rate, set one by default.
 455   if (source_frame_rate == 0.0f)
 456     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 457   source_frame_rate_ = source_frame_rate;
 458   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 459       << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 460   io_message_loop_->PostDelayedTask(FROM_HERE,
 461        base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 462                   on_muted_callback, frame_counter_),
 463        base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals /
 464                                      source_frame_rate_));
 465 }
 466
 467 void VideoTrackAdapter::StopFrameMonitoringOnIO() {
 468   DCHECK(io_message_loop_->BelongsToCurrentThread());
 469   monitoring_frame_rate_ = false;
 470 }
 471
 472 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
 473   DCHECK(io_message_loop_->BelongsToCurrentThread());
 474   for (FrameAdapters::iterator it = adapters_.begin();
 475        it != adapters_.end(); ++it) {
 476     (*it)->RemoveCallback(track);
 477     if ((*it)->IsEmpty()) {
 478       adapters_.erase(it);
 479       break;
 480     }
 481   }
 482 }
 483
 484 void VideoTrackAdapter::DeliverFrameOnIO(
 485     const scoped_refptr<media::VideoFrame>& frame,
 486     const base::TimeTicks& estimated_capture_time) {
 487   DCHECK(io_message_loop_->BelongsToCurrentThread());
 488   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
 489   ++frame_counter_;
 490   for (const auto& adapter : adapters_)
 491     adapter->DeliverFrame(frame, estimated_capture_time);
 492 }
 493
 494 void VideoTrackAdapter::CheckFramesReceivedOnIO(
 495     const OnMutedCallback& set_muted_state_callback,
 496     uint64 old_frame_counter_snapshot) {
 497   DCHECK(io_message_loop_->BelongsToCurrentThread());
 498
 499   if (!monitoring_frame_rate_)
 500     return;
 501
 502   DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_)
 503       << "No frames have passed, setting source as Muted.";
 504
 505   bool muted_state = old_frame_counter_snapshot == frame_counter_;
 506   if (muted_state_ != muted_state) {
 507     set_muted_state_callback.Run(muted_state);
 508     muted_state_ = muted_state;
 509   }
 510
 511   io_message_loop_->PostDelayedTask(FROM_HERE,
 512       base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 513           set_muted_state_callback, frame_counter_),
 514       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
 515                                     source_frame_rate_));
 516 }
 517
 518 }  // namespace content