content/renderer/media/video_track_adapter.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_adapter.h"
   6
   7 #include <algorithm>
   8 #include <limits>
   9 #include <utility>
  10
  11 #include "base/bind.h"
  12 #include "base/command_line.h"
  13 #include "base/location.h"
  14 #include "base/metrics/histogram.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/trace_event/trace_event.h"
  17 #include "content/public/common/content_switches.h"
  18 #include "media/base/bind_to_current_loop.h"
  19 #include "media/base/video_util.h"
  20
  21 namespace content {
  22
  23 namespace {
  24
  25 // Amount of frame intervals to wait before considering the source as muted, for
  26 // the first frame and under normal conditions, respectively. First frame might
  27 // take longer to arrive due to source startup.
  28 const float kFirstFrameTimeoutInFrameIntervals = 100.0f;
  29 const float kNormalFrameTimeoutInFrameIntervals = 25.0f;
  30
  31 // Min delta time between two frames allowed without being dropped if a max
  32 // frame rate is specified.
  33 const double kMinTimeInMsBetweenFrames = 5;
  34 // If the delta between two frames is bigger than this, we will consider it to
  35 // be invalid and reset the fps calculation.
  36 const double kMaxTimeInMsBetweenFrames = 1000;
  37
  38 // Empty method used for keeping a reference to the original media::VideoFrame
  39 // in VideoFrameResolutionAdapter::DeliverFrame if cropping is needed.
  40 // The reference to |frame| is kept in the closure that calls this method.
  41 void ReleaseOriginalFrame(const scoped_refptr<media::VideoFrame>& frame) {
  42 }
  43
  44 void ResetCallbackOnMainRenderThread(
  45     scoped_ptr<VideoCaptureDeliverFrameCB> callback) {
  46   // |callback| will be deleted when this exits.
  47 }
  48
  49 }  // anonymous namespace
  50
  51 // VideoFrameResolutionAdapter is created on and lives on the IO-thread. It does
  52 // the resolution adaptation and delivers frames to all registered tracks on the
  53 // IO-thread. All method calls must be on the IO-thread.
  54 class VideoTrackAdapter::VideoFrameResolutionAdapter
  55     : public base::RefCountedThreadSafe<VideoFrameResolutionAdapter> {
  56  public:
  57   // Setting |max_frame_rate| to 0.0, means that no frame rate limitation
  58   // will be done.
  59   VideoFrameResolutionAdapter(
  60       scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
  61       const gfx::Size& max_size,
  62       double min_aspect_ratio,
  63       double max_aspect_ratio,
  64       double max_frame_rate);
  65
  66   // Add |callback| to receive video frames on the IO-thread.
  67   // |callback| will however be released on the main render thread.
  68   void AddCallback(const MediaStreamVideoTrack* track,
  69                    const VideoCaptureDeliverFrameCB& callback);
  70
  71   // Removes |callback| associated with |track| from receiving video frames if
  72   // |track| has been added. It is ok to call RemoveCallback even if the |track|
  73   // has not been added. The |callback| is released on the main render thread.
  74   void RemoveCallback(const MediaStreamVideoTrack* track);
  75
  76   void DeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  77                     const base::TimeTicks& estimated_capture_time);
  78
  79   // Returns true if all arguments match with the output of this adapter.
  80   bool ConstraintsMatch(const gfx::Size& max_size,
  81                         double min_aspect_ratio,
  82                         double max_aspect_ratio,
  83                         double max_frame_rate) const;
  84
  85   bool IsEmpty() const;
  86
  87  private:
  88   virtual ~VideoFrameResolutionAdapter();
  89   friend class base::RefCountedThreadSafe<VideoFrameResolutionAdapter>;
  90
  91   void DoDeliverFrame(const scoped_refptr<media::VideoFrame>& frame,
  92                       const base::TimeTicks& estimated_capture_time);
  93
  94   // Returns |true| if the input frame rate is higher that the requested max
  95   // frame rate and |frame| should be dropped.
  96   bool MaybeDropFrame(const scoped_refptr<media::VideoFrame>& frame,
  97                       float source_frame_rate);
  98
  99   // Bound to the IO-thread.
 100   base::ThreadChecker io_thread_checker_;
 101
 102   // The task runner where we will release VideoCaptureDeliverFrameCB
 103   // registered in AddCallback.
 104   const scoped_refptr<base::SingleThreadTaskRunner> renderer_task_runner_;
 105
 106   const gfx::Size max_frame_size_;
 107   const double min_aspect_ratio_;
 108   const double max_aspect_ratio_;
 109
 110   double frame_rate_;
 111   base::TimeDelta last_time_stamp_;
 112   double max_frame_rate_;
 113   double keep_frame_counter_;
 114
 115   typedef std::pair<const MediaStreamVideoTrack*, VideoCaptureDeliverFrameCB>
 116       VideoIdCallbackPair;
 117   std::vector<VideoIdCallbackPair> callbacks_;
 118
 119   DISALLOW_COPY_AND_ASSIGN(VideoFrameResolutionAdapter);
 120 };
 121
 122 VideoTrackAdapter::
 123 VideoFrameResolutionAdapter::VideoFrameResolutionAdapter(
 124     scoped_refptr<base::SingleThreadTaskRunner> render_message_loop,
 125     const gfx::Size& max_size,
 126     double min_aspect_ratio,
 127     double max_aspect_ratio,
 128     double max_frame_rate)
 129     : renderer_task_runner_(render_message_loop),
 130       max_frame_size_(max_size),
 131       min_aspect_ratio_(min_aspect_ratio),
 132       max_aspect_ratio_(max_aspect_ratio),
 133       frame_rate_(MediaStreamVideoSource::kDefaultFrameRate),
 134       last_time_stamp_(base::TimeDelta::Max()),
 135       max_frame_rate_(max_frame_rate),
 136       keep_frame_counter_(0.0) {
 137   DCHECK(renderer_task_runner_.get());
 138   DCHECK(io_thread_checker_.CalledOnValidThread());
 139   DCHECK_GE(max_aspect_ratio_, min_aspect_ratio_);
 140   CHECK_NE(0, max_aspect_ratio_);
 141
 142   const std::string max_fps_str =
 143       base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
 144           switches::kWebRtcMaxCaptureFramerate);
 145   if (!max_fps_str.empty()) {
 146     double value;
 147     if (base::StringToDouble(max_fps_str, &value) && value >= 0.0) {
 148       DVLOG(1) << "Overriding max frame rate.  Was=" << max_frame_rate
 149               << ", Now=" << value;
 150       max_frame_rate_ = value;
 151     } else {
 152       DLOG(ERROR) << "Unable to set max fps to " << max_fps_str;
 153     }
 154   }
 155
 156   DVLOG(3) << "VideoFrameResolutionAdapter("
 157           << "{ max_width =" << max_frame_size_.width() << "}, "
 158           << "{ max_height =" << max_frame_size_.height() << "}, "
 159           << "{ min_aspect_ratio =" << min_aspect_ratio << "}, "
 160           << "{ max_aspect_ratio_ =" << max_aspect_ratio_ << "}"
 161           << "{ max_frame_rate_ =" << max_frame_rate_ << "}) ";
 162 }
 163
 164 VideoTrackAdapter::
 165 VideoFrameResolutionAdapter::~VideoFrameResolutionAdapter() {
 166   DCHECK(io_thread_checker_.CalledOnValidThread());
 167   DCHECK(callbacks_.empty());
 168 }
 169
 170 void VideoTrackAdapter::VideoFrameResolutionAdapter::AddCallback(
 171     const MediaStreamVideoTrack* track,
 172     const VideoCaptureDeliverFrameCB& callback) {
 173   DCHECK(io_thread_checker_.CalledOnValidThread());
 174   callbacks_.push_back(std::make_pair(track, callback));
 175 }
 176
 177 void VideoTrackAdapter::VideoFrameResolutionAdapter::RemoveCallback(
 178     const MediaStreamVideoTrack* track) {
 179   DCHECK(io_thread_checker_.CalledOnValidThread());
 180   std::vector<VideoIdCallbackPair>::iterator it = callbacks_.begin();
 181   for (; it != callbacks_.end(); ++it) {
 182     if (it->first == track) {
 183       // Make sure the VideoCaptureDeliverFrameCB is released on the main
 184       // render thread since it was added on the main render thread in
 185       // VideoTrackAdapter::AddTrack.
 186       scoped_ptr<VideoCaptureDeliverFrameCB> callback(
 187           new VideoCaptureDeliverFrameCB(it->second));
 188       callbacks_.erase(it);
 189       renderer_task_runner_->PostTask(
 190           FROM_HERE, base::Bind(&ResetCallbackOnMainRenderThread,
 191                                 base::Passed(&callback)));
 192
 193       return;
 194     }
 195   }
 196 }
 197
 198 void VideoTrackAdapter::VideoFrameResolutionAdapter::DeliverFrame(
 199     const scoped_refptr<media::VideoFrame>& frame,
 200     const base::TimeTicks& estimated_capture_time) {
 201   DCHECK(io_thread_checker_.CalledOnValidThread());
 202
 203   double frame_rate;
 204   if (!frame->metadata()->GetDouble(media::VideoFrameMetadata::FRAME_RATE,
 205                                     &frame_rate)) {
 206     frame_rate = MediaStreamVideoSource::kUnknownFrameRate;
 207   }
 208
 209   if (MaybeDropFrame(frame, frame_rate))
 210     return;
 211
 212   // TODO(perkj): Allow cropping / scaling of textures once
 213   // http://crbug/362521 is fixed.
 214   if (frame->HasTextures()) {
 215     DoDeliverFrame(frame, estimated_capture_time);
 216     return;
 217   }
 218   scoped_refptr<media::VideoFrame> video_frame(frame);
 219   const double input_ratio =
 220       static_cast<double>(frame->natural_size().width()) /
 221       frame->natural_size().height();
 222
 223   // If |frame| has larger width or height than requested, or the aspect ratio
 224   // does not match the requested, we want to create a wrapped version of this
 225   // frame with a size that fulfills the constraints.
 226   if (frame->natural_size().width() > max_frame_size_.width() ||
 227       frame->natural_size().height() > max_frame_size_.height() ||
 228       input_ratio > max_aspect_ratio_ ||
 229       input_ratio < min_aspect_ratio_) {
 230     int desired_width = std::min(max_frame_size_.width(),
 231                                  frame->natural_size().width());
 232     int desired_height = std::min(max_frame_size_.height(),
 233                                   frame->natural_size().height());
 234
 235     const double resulting_ratio =
 236         static_cast<double>(desired_width) / desired_height;
 237     // Make sure |min_aspect_ratio_| < |requested_ratio| < |max_aspect_ratio_|.
 238     const double requested_ratio = std::max(
 239         std::min(resulting_ratio, max_aspect_ratio_), min_aspect_ratio_);
 240
 241     if (resulting_ratio < requested_ratio) {
 242       desired_height = static_cast<int>((desired_height * resulting_ratio) /
 243                                         requested_ratio);
 244       // Make sure we scale to an even height to avoid rounding errors
 245       desired_height = (desired_height + 1) & ~1;
 246     } else if (resulting_ratio > requested_ratio) {
 247       desired_width = static_cast<int>((desired_width * requested_ratio) /
 248                                        resulting_ratio);
 249       // Make sure we scale to an even width to avoid rounding errors.
 250       desired_width = (desired_width + 1) & ~1;
 251     }
 252
 253     const gfx::Size desired_size(desired_width, desired_height);
 254
 255     // Get the largest centered rectangle with the same aspect ratio of
 256     // |desired_size| that fits entirely inside of |frame->visible_rect()|.
 257     // This will be the rect we need to crop the original frame to.
 258     // From this rect, the original frame can be scaled down to |desired_size|.
 259     const gfx::Rect region_in_frame =
 260         media::ComputeLetterboxRegion(frame->visible_rect(), desired_size);
 261
 262     video_frame =
 263         media::VideoFrame::WrapVideoFrame(frame, region_in_frame, desired_size);
 264     video_frame->AddDestructionObserver(
 265         base::Bind(&ReleaseOriginalFrame, frame));
 266
 267     DVLOG(3) << "desired size  " << desired_size.ToString()
 268              << " output natural size "
 269              << video_frame->natural_size().ToString()
 270              << " output visible rect  "
 271              << video_frame->visible_rect().ToString();
 272   }
 273   DoDeliverFrame(video_frame, estimated_capture_time);
 274 }
 275
 276 bool VideoTrackAdapter::VideoFrameResolutionAdapter::ConstraintsMatch(
 277     const gfx::Size& max_size,
 278     double min_aspect_ratio,
 279     double max_aspect_ratio,
 280     double max_frame_rate) const {
 281   DCHECK(io_thread_checker_.CalledOnValidThread());
 282   return max_frame_size_ == max_size &&
 283       min_aspect_ratio_ == min_aspect_ratio &&
 284       max_aspect_ratio_ == max_aspect_ratio &&
 285       max_frame_rate_ == max_frame_rate;
 286 }
 287
 288 bool VideoTrackAdapter::VideoFrameResolutionAdapter::IsEmpty() const {
 289   DCHECK(io_thread_checker_.CalledOnValidThread());
 290   return callbacks_.empty();
 291 }
 292
 293 void VideoTrackAdapter::VideoFrameResolutionAdapter::DoDeliverFrame(
 294     const scoped_refptr<media::VideoFrame>& frame,
 295     const base::TimeTicks& estimated_capture_time) {
 296   DCHECK(io_thread_checker_.CalledOnValidThread());
 297   for (const auto& callback : callbacks_)
 298     callback.second.Run(frame, estimated_capture_time);
 299 }
 300
 301 bool VideoTrackAdapter::VideoFrameResolutionAdapter::MaybeDropFrame(
 302     const scoped_refptr<media::VideoFrame>& frame,
 303     float source_frame_rate) {
 304   DCHECK(io_thread_checker_.CalledOnValidThread());
 305
 306   // Do not drop frames if max frame rate hasn't been specified or the source
 307   // frame rate is known and is lower than max.
 308   if (max_frame_rate_ == 0.0f ||
 309       (source_frame_rate > 0 && source_frame_rate <= max_frame_rate_)) {
 310     return false;
 311   }
 312
 313   const double delta_ms =
 314       (frame->timestamp() - last_time_stamp_).InMillisecondsF();
 315
 316   // Check if the time since the last frame is completely off.
 317   if (delta_ms < 0 || delta_ms > kMaxTimeInMsBetweenFrames) {
 318     // Reset |last_time_stamp_| and fps calculation.
 319     last_time_stamp_ = frame->timestamp();
 320     frame_rate_ = MediaStreamVideoSource::kDefaultFrameRate;
 321     keep_frame_counter_ = 0.0;
 322     return false;
 323   }
 324
 325   if (delta_ms < kMinTimeInMsBetweenFrames) {
 326     // We have seen video frames being delivered from camera devices back to
 327     // back. The simple AR filter for frame rate calculation is too short to
 328     // handle that. http://crbug/394315
 329     // TODO(perkj): Can we come up with a way to fix the times stamps and the
 330     // timing when frames are delivered so all frames can be used?
 331     // The time stamps are generated by Chrome and not the actual device.
 332     // Most likely the back to back problem is caused by software and not the
 333     // actual camera.
 334     DVLOG(3) << "Drop frame since delta time since previous frame is "
 335              << delta_ms << "ms.";
 336     return true;
 337   }
 338   last_time_stamp_ = frame->timestamp();
 339   // Calculate the frame rate using a simple AR filter.
 340   // Use a simple filter with 0.1 weight of the current sample.
 341   frame_rate_ = 100 / delta_ms + 0.9 * frame_rate_;
 342
 343   // Prefer to not drop frames.
 344   if (max_frame_rate_ + 0.5f > frame_rate_)
 345     return false;  // Keep this frame.
 346
 347   // The input frame rate is higher than requested.
 348   // Decide if we should keep this frame or drop it.
 349   keep_frame_counter_ += max_frame_rate_ / frame_rate_;
 350   if (keep_frame_counter_ >= 1) {
 351     keep_frame_counter_ -= 1;
 352     // Keep the frame.
 353     return false;
 354   }
 355   DVLOG(3) << "Drop frame. Input frame_rate_ " << frame_rate_ << ".";
 356   return true;
 357 }
 358
 359 VideoTrackAdapter::VideoTrackAdapter(
 360     scoped_refptr<base::SingleThreadTaskRunner> io_task_runner)
 361     : io_task_runner_(io_task_runner),
 362       renderer_task_runner_(base::ThreadTaskRunnerHandle::Get()),
 363       monitoring_frame_rate_(false),
 364       muted_state_(false),
 365       frame_counter_(0),
 366       source_frame_rate_(0.0f) {
 367   DCHECK(io_task_runner);
 368 }
 369
 370 VideoTrackAdapter::~VideoTrackAdapter() {
 371   DCHECK(adapters_.empty());
 372 }
 373
 374 void VideoTrackAdapter::AddTrack(const MediaStreamVideoTrack* track,
 375                                  VideoCaptureDeliverFrameCB frame_callback,
 376                                  int max_width,
 377                                  int max_height,
 378                                  double min_aspect_ratio,
 379                                  double max_aspect_ratio,
 380                                  double max_frame_rate) {
 381   DCHECK(thread_checker_.CalledOnValidThread());
 382
 383   io_task_runner_->PostTask(
 384       FROM_HERE,
 385       base::Bind(&VideoTrackAdapter::AddTrackOnIO, this, track, frame_callback,
 386                  gfx::Size(max_width, max_height), min_aspect_ratio,
 387                  max_aspect_ratio, max_frame_rate));
 388 }
 389
 390 void VideoTrackAdapter::AddTrackOnIO(const MediaStreamVideoTrack* track,
 391                                      VideoCaptureDeliverFrameCB frame_callback,
 392                                      const gfx::Size& max_frame_size,
 393                                      double min_aspect_ratio,
 394                                      double max_aspect_ratio,
 395                                      double max_frame_rate) {
 396   DCHECK(io_task_runner_->BelongsToCurrentThread());
 397   scoped_refptr<VideoFrameResolutionAdapter> adapter;
 398   for (const auto& frame_adapter : adapters_) {
 399     if (frame_adapter->ConstraintsMatch(max_frame_size, min_aspect_ratio,
 400                                         max_aspect_ratio, max_frame_rate)) {
 401       adapter = frame_adapter.get();
 402       break;
 403     }
 404   }
 405   if (!adapter.get()) {
 406     adapter = new VideoFrameResolutionAdapter(renderer_task_runner_,
 407                                               max_frame_size,
 408                                               min_aspect_ratio,
 409                                               max_aspect_ratio,
 410                                               max_frame_rate);
 411     adapters_.push_back(adapter);
 412   }
 413
 414   adapter->AddCallback(track, frame_callback);
 415 }
 416
 417 void VideoTrackAdapter::RemoveTrack(const MediaStreamVideoTrack* track) {
 418   DCHECK(thread_checker_.CalledOnValidThread());
 419   io_task_runner_->PostTask(
 420       FROM_HERE, base::Bind(&VideoTrackAdapter::RemoveTrackOnIO, this, track));
 421 }
 422
 423 void VideoTrackAdapter::StartFrameMonitoring(
 424     double source_frame_rate,
 425     const OnMutedCallback& on_muted_callback) {
 426   DCHECK(thread_checker_.CalledOnValidThread());
 427
 428   VideoTrackAdapter::OnMutedCallback bound_on_muted_callback =
 429       media::BindToCurrentLoop(on_muted_callback);
 430
 431   io_task_runner_->PostTask(
 432       FROM_HERE, base::Bind(&VideoTrackAdapter::StartFrameMonitoringOnIO, this,
 433                             bound_on_muted_callback, source_frame_rate));
 434 }
 435
 436 void VideoTrackAdapter::StopFrameMonitoring() {
 437   DCHECK(thread_checker_.CalledOnValidThread());
 438   io_task_runner_->PostTask(
 439       FROM_HERE, base::Bind(&VideoTrackAdapter::StopFrameMonitoringOnIO, this));
 440 }
 441
 442 void VideoTrackAdapter::StartFrameMonitoringOnIO(
 443     const OnMutedCallback& on_muted_callback,
 444     double source_frame_rate) {
 445   DCHECK(io_task_runner_->BelongsToCurrentThread());
 446   DCHECK(!monitoring_frame_rate_);
 447
 448   monitoring_frame_rate_ = true;
 449
 450   // If the source does not know the frame rate, set one by default.
 451   if (source_frame_rate == 0.0f)
 452     source_frame_rate = MediaStreamVideoSource::kDefaultFrameRate;
 453   source_frame_rate_ = source_frame_rate;
 454   DVLOG(1) << "Monitoring frame creation, first (large) delay: "
 455       << (kFirstFrameTimeoutInFrameIntervals / source_frame_rate_) << "s";
 456   io_task_runner_->PostDelayedTask(
 457       FROM_HERE, base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 458                             on_muted_callback, frame_counter_),
 459       base::TimeDelta::FromSecondsD(kFirstFrameTimeoutInFrameIntervals /
 460                                     source_frame_rate_));
 461 }
 462
 463 void VideoTrackAdapter::StopFrameMonitoringOnIO() {
 464   DCHECK(io_task_runner_->BelongsToCurrentThread());
 465   monitoring_frame_rate_ = false;
 466 }
 467
 468 void VideoTrackAdapter::RemoveTrackOnIO(const MediaStreamVideoTrack* track) {
 469   DCHECK(io_task_runner_->BelongsToCurrentThread());
 470   for (FrameAdapters::iterator it = adapters_.begin();
 471        it != adapters_.end(); ++it) {
 472     (*it)->RemoveCallback(track);
 473     if ((*it)->IsEmpty()) {
 474       adapters_.erase(it);
 475       break;
 476     }
 477   }
 478 }
 479
 480 void VideoTrackAdapter::DeliverFrameOnIO(
 481     const scoped_refptr<media::VideoFrame>& frame,
 482     base::TimeTicks estimated_capture_time) {
 483   DCHECK(io_task_runner_->BelongsToCurrentThread());
 484   TRACE_EVENT0("video", "VideoTrackAdapter::DeliverFrameOnIO");
 485   ++frame_counter_;
 486   for (const auto& adapter : adapters_)
 487     adapter->DeliverFrame(frame, estimated_capture_time);
 488 }
 489
 490 void VideoTrackAdapter::CheckFramesReceivedOnIO(
 491     const OnMutedCallback& set_muted_state_callback,
 492     uint64 old_frame_counter_snapshot) {
 493   DCHECK(io_task_runner_->BelongsToCurrentThread());
 494
 495   if (!monitoring_frame_rate_)
 496     return;
 497
 498   DVLOG_IF(1, old_frame_counter_snapshot == frame_counter_)
 499       << "No frames have passed, setting source as Muted.";
 500
 501   bool muted_state = old_frame_counter_snapshot == frame_counter_;
 502   if (muted_state_ != muted_state) {
 503     set_muted_state_callback.Run(muted_state);
 504     muted_state_ = muted_state;
 505   }
 506
 507   io_task_runner_->PostDelayedTask(
 508       FROM_HERE, base::Bind(&VideoTrackAdapter::CheckFramesReceivedOnIO, this,
 509                             set_muted_state_callback, frame_counter_),
 510       base::TimeDelta::FromSecondsD(kNormalFrameTimeoutInFrameIntervals /
 511                                     source_frame_rate_));
 512 }
 513
 514 }  // namespace content