media/capture/content/video_capture_oracle.cc

   1 // Copyright (c) 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/capture/content/video_capture_oracle.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/format_macros.h"
  10 #include "base/numerics/safe_conversions.h"
  11 #include "base/strings/stringprintf.h"
  12
  13 namespace media {
  14
  15 namespace {
  16
  17 // This value controls how many redundant, timer-base captures occur when the
  18 // content is static. Redundantly capturing the same frame allows iterative
  19 // quality enhancement, and also allows the buffer to fill in "buffered mode".
  20 //
  21 // TODO(nick): Controlling this here is a hack and a layering violation, since
  22 // it's a strategy specific to the WebRTC consumer, and probably just papers
  23 // over some frame dropping and quality bugs. It should either be controlled at
  24 // a higher level, or else redundant frame generation should be pushed down
  25 // further into the WebRTC encoding stack.
  26 const int kNumRedundantCapturesOfStaticContent = 200;
  27
  28 // The half-life of data points provided to the accumulator used when evaluating
  29 // the recent utilization of the buffer pool.  This value is based on a
  30 // simulation, and reacts quickly to change to avoid depleting the buffer pool
  31 // (which would cause hard frame drops).
  32 const int kBufferUtilizationEvaluationMicros = 200000;  // 0.2 seconds
  33
  34 // The half-life of data points provided to the accumulator used when evaluating
  35 // the recent resource utilization of the consumer.  The trade-off made here is
  36 // reaction time versus over-reacting to outlier data points.
  37 const int kConsumerCapabilityEvaluationMicros = 1000000;  // 1 second
  38
  39 // The minimum amount of time that must pass between changes to the capture
  40 // size.  This throttles the rate of size changes, to avoid stressing consumers
  41 // and to allow the end-to-end system sufficient time to stabilize before
  42 // re-evaluating the capture size.
  43 const int kMinSizeChangePeriodMicros = 3000000;  // 3 seconds
  44
  45 // The maximum amount of time that may elapse without a feedback update.  Any
  46 // longer, and currently-accumulated feedback is not considered recent enough to
  47 // base decisions off of.  This prevents changes to the capture size when there
  48 // is an unexpected pause in events.
  49 const int kMaxTimeSinceLastFeedbackUpdateMicros = 1000000;  // 1 second
  50
  51 // The amount of additional time, since content animation was last detected, to
  52 // continue being extra-careful about increasing the capture size.  This is used
  53 // to prevent breif periods of non-animating content from throwing off the
  54 // heuristics that decide whether to increase the capture size.
  55 const int kDebouncingPeriodForAnimatedContentMicros = 3000000;  // 3 seconds
  56
  57 // When content is animating, this is the length of time the system must be
  58 // contiguously under-utilized before increasing the capture size.
  59 const int kProvingPeriodForAnimatedContentMicros = 30000000;  // 30 seconds
  60
  61 // Given the amount of time between frames, compare to the expected amount of
  62 // time between frames at |frame_rate| and return the fractional difference.
  63 double FractionFromExpectedFrameRate(base::TimeDelta delta, int frame_rate) {
  64   DCHECK_GT(frame_rate, 0);
  65   const base::TimeDelta expected_delta =
  66       base::TimeDelta::FromSeconds(1) / frame_rate;
  67   return (delta - expected_delta).InMillisecondsF() /
  68          expected_delta.InMillisecondsF();
  69 }
  70
  71 // Returns the next-higher TimeTicks value.
  72 // TODO(miu): Patch FeedbackSignalAccumulator reset behavior and remove this
  73 // hack.
  74 base::TimeTicks JustAfter(base::TimeTicks t) {
  75   return t + base::TimeDelta::FromMicroseconds(1);
  76 }
  77
  78 // Returns true if updates have been accumulated by |accumulator| for a
  79 // sufficient amount of time and the latest update was fairly recent, relative
  80 // to |now|.
  81 bool HasSufficientRecentFeedback(const FeedbackSignalAccumulator& accumulator,
  82                                  base::TimeTicks now) {
  83   const base::TimeDelta amount_of_history =
  84       accumulator.update_time() - accumulator.reset_time();
  85   return (amount_of_history.InMicroseconds() >= kMinSizeChangePeriodMicros) &&
  86          ((now - accumulator.update_time()).InMicroseconds() <=
  87           kMaxTimeSinceLastFeedbackUpdateMicros);
  88 }
  89
  90 }  // anonymous namespace
  91
  92 VideoCaptureOracle::VideoCaptureOracle(
  93     base::TimeDelta min_capture_period,
  94     const gfx::Size& max_frame_size,
  95     media::ResolutionChangePolicy resolution_change_policy,
  96     bool enable_auto_throttling)
  97     : auto_throttling_enabled_(enable_auto_throttling),
  98       next_frame_number_(0),
  99       last_successfully_delivered_frame_number_(-1),
 100       num_frames_pending_(0),
 101       smoothing_sampler_(min_capture_period,
 102                          kNumRedundantCapturesOfStaticContent),
 103       content_sampler_(min_capture_period),
 104       resolution_chooser_(max_frame_size, resolution_change_policy),
 105       buffer_pool_utilization_(base::TimeDelta::FromMicroseconds(
 106           kBufferUtilizationEvaluationMicros)),
 107       estimated_capable_area_(base::TimeDelta::FromMicroseconds(
 108           kConsumerCapabilityEvaluationMicros)) {
 109   VLOG(1) << "Auto-throttling is "
 110           << (auto_throttling_enabled_ ? "enabled." : "disabled.");
 111 }
 112
 113 VideoCaptureOracle::~VideoCaptureOracle() {
 114 }
 115
 116 void VideoCaptureOracle::SetSourceSize(const gfx::Size& source_size) {
 117   resolution_chooser_.SetSourceSize(source_size);
 118   // If the |resolution_chooser_| computed a new capture size, that will become
 119   // visible via a future call to ObserveEventAndDecideCapture().
 120 }
 121
 122 bool VideoCaptureOracle::ObserveEventAndDecideCapture(
 123     Event event,
 124     const gfx::Rect& damage_rect,
 125     base::TimeTicks event_time) {
 126   DCHECK_GE(event, 0);
 127   DCHECK_LT(event, kNumEvents);
 128   if (event_time < last_event_time_[event]) {
 129     LOG(WARNING) << "Event time is not monotonically non-decreasing.  "
 130                  << "Deciding not to capture this frame.";
 131     return false;
 132   }
 133   last_event_time_[event] = event_time;
 134
 135   bool should_sample = false;
 136   duration_of_next_frame_ = base::TimeDelta();
 137   switch (event) {
 138     case kCompositorUpdate: {
 139       smoothing_sampler_.ConsiderPresentationEvent(event_time);
 140       const bool had_proposal = content_sampler_.HasProposal();
 141       content_sampler_.ConsiderPresentationEvent(damage_rect, event_time);
 142       if (content_sampler_.HasProposal()) {
 143         VLOG_IF(1, !had_proposal) << "Content sampler now detects animation.";
 144         should_sample = content_sampler_.ShouldSample();
 145         if (should_sample) {
 146           event_time = content_sampler_.frame_timestamp();
 147           duration_of_next_frame_ = content_sampler_.sampling_period();
 148         }
 149         last_time_animation_was_detected_ = event_time;
 150       } else {
 151         VLOG_IF(1, had_proposal) << "Content sampler detects animation ended.";
 152         should_sample = smoothing_sampler_.ShouldSample();
 153         if (should_sample)
 154           duration_of_next_frame_ = smoothing_sampler_.min_capture_period();
 155       }
 156       break;
 157     }
 158
 159     case kTimerPoll:
 160       // While the timer is firing, only allow a sampling if there are none
 161       // currently in-progress.
 162       if (num_frames_pending_ == 0) {
 163         should_sample = smoothing_sampler_.IsOverdueForSamplingAt(event_time);
 164         if (should_sample)
 165           duration_of_next_frame_ = smoothing_sampler_.min_capture_period();
 166       }
 167       break;
 168
 169     case kNumEvents:
 170       NOTREACHED();
 171       break;
 172   }
 173
 174   if (!should_sample)
 175     return false;
 176
 177   // Update |capture_size_| and reset all feedback signal accumulators if
 178   // either: 1) this is the first frame; or 2) |resolution_chooser_| has an
 179   // updated capture size and sufficient time has passed since the last size
 180   // change.
 181   if (next_frame_number_ == 0) {
 182     CommitCaptureSizeAndReset(event_time - duration_of_next_frame_);
 183   } else if (capture_size_ != resolution_chooser_.capture_size()) {
 184     const base::TimeDelta time_since_last_change =
 185         event_time - buffer_pool_utilization_.reset_time();
 186     if (time_since_last_change.InMicroseconds() >= kMinSizeChangePeriodMicros)
 187       CommitCaptureSizeAndReset(GetFrameTimestamp(next_frame_number_ - 1));
 188   }
 189
 190   SetFrameTimestamp(next_frame_number_, event_time);
 191   return true;
 192 }
 193
 194 int VideoCaptureOracle::RecordCapture(double pool_utilization) {
 195   DCHECK(std::isfinite(pool_utilization) && pool_utilization >= 0.0);
 196
 197   smoothing_sampler_.RecordSample();
 198   const base::TimeTicks timestamp = GetFrameTimestamp(next_frame_number_);
 199   content_sampler_.RecordSample(timestamp);
 200
 201   if (auto_throttling_enabled_) {
 202     buffer_pool_utilization_.Update(pool_utilization, timestamp);
 203     AnalyzeAndAdjust(timestamp);
 204   }
 205
 206   num_frames_pending_++;
 207   return next_frame_number_++;
 208 }
 209
 210 void VideoCaptureOracle::RecordWillNotCapture(double pool_utilization) {
 211   VLOG(1) << "Client rejects proposal to capture frame (at #"
 212           << next_frame_number_ << ").";
 213
 214   if (auto_throttling_enabled_) {
 215     DCHECK(std::isfinite(pool_utilization) && pool_utilization >= 0.0);
 216     const base::TimeTicks timestamp = GetFrameTimestamp(next_frame_number_);
 217     buffer_pool_utilization_.Update(pool_utilization, timestamp);
 218     AnalyzeAndAdjust(timestamp);
 219   }
 220
 221   // Note: Do not advance |next_frame_number_| since it will be re-used for the
 222   // next capture proposal.
 223 }
 224
 225 bool VideoCaptureOracle::CompleteCapture(int frame_number,
 226                                          bool capture_was_successful,
 227                                          base::TimeTicks* frame_timestamp) {
 228   num_frames_pending_--;
 229   DCHECK_GE(num_frames_pending_, 0);
 230
 231   // Drop frame if previously delivered frame number is higher.
 232   if (last_successfully_delivered_frame_number_ > frame_number) {
 233     LOG_IF(WARNING, capture_was_successful)
 234         << "Out of order frame delivery detected (have #" << frame_number
 235         << ", last was #" << last_successfully_delivered_frame_number_
 236         << ").  Dropping frame.";
 237     return false;
 238   }
 239
 240   if (!IsFrameInRecentHistory(frame_number)) {
 241     LOG(WARNING) << "Very old capture being ignored: frame #" << frame_number;
 242     return false;
 243   }
 244
 245   if (!capture_was_successful) {
 246     VLOG(2) << "Capture of frame #" << frame_number << " was not successful.";
 247     return false;
 248   }
 249
 250   DCHECK_NE(last_successfully_delivered_frame_number_, frame_number);
 251   last_successfully_delivered_frame_number_ = frame_number;
 252
 253   *frame_timestamp = GetFrameTimestamp(frame_number);
 254
 255   // If enabled, log a measurement of how this frame timestamp has incremented
 256   // in relation to an ideal increment.
 257   if (VLOG_IS_ON(3) && frame_number > 0) {
 258     const base::TimeDelta delta =
 259         *frame_timestamp - GetFrameTimestamp(frame_number - 1);
 260     if (content_sampler_.HasProposal()) {
 261       const double estimated_frame_rate =
 262           1000000.0 / content_sampler_.detected_period().InMicroseconds();
 263       const int rounded_frame_rate =
 264           static_cast<int>(estimated_frame_rate + 0.5);
 265       VLOG_STREAM(3) << base::StringPrintf(
 266           "Captured #%d: delta=%" PRId64
 267           " usec"
 268           ", now locked into {%s}, %+0.1f%% slower than %d FPS",
 269           frame_number, delta.InMicroseconds(),
 270           content_sampler_.detected_region().ToString().c_str(),
 271           100.0 * FractionFromExpectedFrameRate(delta, rounded_frame_rate),
 272           rounded_frame_rate);
 273     } else {
 274       VLOG_STREAM(3) << base::StringPrintf(
 275           "Captured #%d: delta=%" PRId64
 276           " usec"
 277           ", d/30fps=%+0.1f%%, d/25fps=%+0.1f%%, d/24fps=%+0.1f%%",
 278           frame_number, delta.InMicroseconds(),
 279           100.0 * FractionFromExpectedFrameRate(delta, 30),
 280           100.0 * FractionFromExpectedFrameRate(delta, 25),
 281           100.0 * FractionFromExpectedFrameRate(delta, 24));
 282     }
 283   }
 284
 285   return true;
 286 }
 287
 288 void VideoCaptureOracle::RecordConsumerFeedback(int frame_number,
 289                                                 double resource_utilization) {
 290   if (!auto_throttling_enabled_)
 291     return;
 292
 293   if (!std::isfinite(resource_utilization)) {
 294     LOG(DFATAL) << "Non-finite utilization provided by consumer for frame #"
 295                 << frame_number << ": " << resource_utilization;
 296     return;
 297   }
 298   if (resource_utilization <= 0.0)
 299     return;  // Non-positive values are normal, meaning N/A.
 300
 301   if (!IsFrameInRecentHistory(frame_number)) {
 302     VLOG(1) << "Very old frame feedback being ignored: frame #" << frame_number;
 303     return;
 304   }
 305   const base::TimeTicks timestamp = GetFrameTimestamp(frame_number);
 306
 307   // Translate the utilization metric to be in terms of the capable frame area
 308   // and update the feedback accumulators.  Research suggests utilization is at
 309   // most linearly proportional to area, and typically is sublinear.  Either
 310   // way, the end-to-end system should converge to the right place using the
 311   // more-conservative assumption (linear).
 312   const int area_at_full_utilization =
 313       base::saturated_cast<int>(capture_size_.GetArea() / resource_utilization);
 314   estimated_capable_area_.Update(area_at_full_utilization, timestamp);
 315 }
 316
 317 base::TimeTicks VideoCaptureOracle::GetFrameTimestamp(int frame_number) const {
 318   DCHECK(IsFrameInRecentHistory(frame_number));
 319   return frame_timestamps_[frame_number % kMaxFrameTimestamps];
 320 }
 321
 322 void VideoCaptureOracle::SetFrameTimestamp(int frame_number,
 323                                            base::TimeTicks timestamp) {
 324   DCHECK(IsFrameInRecentHistory(frame_number));
 325   frame_timestamps_[frame_number % kMaxFrameTimestamps] = timestamp;
 326 }
 327
 328 bool VideoCaptureOracle::IsFrameInRecentHistory(int frame_number) const {
 329   return ((next_frame_number_ - frame_number) < kMaxFrameTimestamps &&
 330           frame_number <= next_frame_number_ && frame_number >= 0);
 331 }
 332
 333 void VideoCaptureOracle::CommitCaptureSizeAndReset(
 334     base::TimeTicks last_frame_time) {
 335   capture_size_ = resolution_chooser_.capture_size();
 336   VLOG(2) << "Now proposing a capture size of " << capture_size_.ToString();
 337
 338   // Reset each short-term feedback accumulator with a stable-state starting
 339   // value.
 340   const base::TimeTicks ignore_before_time = JustAfter(last_frame_time);
 341   buffer_pool_utilization_.Reset(1.0, ignore_before_time);
 342   estimated_capable_area_.Reset(capture_size_.GetArea(), ignore_before_time);
 343
 344   // With the new capture size, erase any prior conclusion about the end-to-end
 345   // system being under-utilized.
 346   start_time_of_underutilization_ = base::TimeTicks();
 347 }
 348
 349 void VideoCaptureOracle::AnalyzeAndAdjust(const base::TimeTicks analyze_time) {
 350   DCHECK(auto_throttling_enabled_);
 351
 352   const int decreased_area = AnalyzeForDecreasedArea(analyze_time);
 353   if (decreased_area > 0) {
 354     resolution_chooser_.SetTargetFrameArea(decreased_area);
 355     return;
 356   }
 357
 358   const int increased_area = AnalyzeForIncreasedArea(analyze_time);
 359   if (increased_area > 0) {
 360     resolution_chooser_.SetTargetFrameArea(increased_area);
 361     return;
 362   }
 363
 364   // Explicitly set the target frame area to the current capture area.  This
 365   // cancels-out the results of a previous call to this method, where the
 366   // |resolution_chooser_| may have been instructed to increase or decrease the
 367   // capture size.  Conditions may have changed since then which indicate no
 368   // change should be committed (via CommitCaptureSizeAndReset()).
 369   resolution_chooser_.SetTargetFrameArea(capture_size_.GetArea());
 370 }
 371
 372 int VideoCaptureOracle::AnalyzeForDecreasedArea(base::TimeTicks analyze_time) {
 373   const int current_area = capture_size_.GetArea();
 374   DCHECK_GT(current_area, 0);
 375
 376   // Translate the recent-average buffer pool utilization to be in terms of
 377   // "capable number of pixels per frame," for an apples-to-apples comparison
 378   // below.
 379   int buffer_capable_area;
 380   if (HasSufficientRecentFeedback(buffer_pool_utilization_, analyze_time) &&
 381       buffer_pool_utilization_.current() > 1.0) {
 382     // This calculation is hand-wavy, but seems to work well in a variety of
 383     // situations.
 384     buffer_capable_area =
 385         static_cast<int>(current_area / buffer_pool_utilization_.current());
 386   } else {
 387     buffer_capable_area = current_area;
 388   }
 389
 390   int consumer_capable_area;
 391   if (HasSufficientRecentFeedback(estimated_capable_area_, analyze_time)) {
 392     consumer_capable_area =
 393         base::saturated_cast<int>(estimated_capable_area_.current());
 394   } else {
 395     consumer_capable_area = current_area;
 396   }
 397
 398   // If either of the "capable areas" is less than the current capture area,
 399   // decrease the capture area by AT LEAST one step.
 400   int decreased_area = -1;
 401   const int capable_area = std::min(buffer_capable_area, consumer_capable_area);
 402   if (capable_area < current_area) {
 403     decreased_area = std::min(
 404         capable_area,
 405         resolution_chooser_.FindSmallerFrameSize(current_area, 1).GetArea());
 406     start_time_of_underutilization_ = base::TimeTicks();
 407     VLOG(2) << "Proposing a "
 408             << (100.0 * (current_area - decreased_area) / current_area)
 409             << "% decrease in capture area.  :-(";
 410   }
 411
 412   // Always log the capability interpretations at verbose logging level 3.  At
 413   // level 2, only log when when proposing a decreased area.
 414   VLOG(decreased_area == -1 ? 3 : 2)
 415       << "Capability of pool=" << (100.0 * buffer_capable_area / current_area)
 416       << "%, consumer=" << (100.0 * consumer_capable_area / current_area)
 417       << '%';
 418
 419   return decreased_area;
 420 }
 421
 422 int VideoCaptureOracle::AnalyzeForIncreasedArea(base::TimeTicks analyze_time) {
 423   // Compute what one step up in capture size/area would be.  If the current
 424   // area is already at the maximum, no further analysis is necessary.
 425   const int current_area = capture_size_.GetArea();
 426   const int increased_area =
 427       resolution_chooser_.FindLargerFrameSize(current_area, 1).GetArea();
 428   if (increased_area <= current_area)
 429     return -1;
 430
 431   // Determine whether the buffer pool could handle an increase in area.
 432   if (!HasSufficientRecentFeedback(buffer_pool_utilization_, analyze_time))
 433     return -1;
 434   if (buffer_pool_utilization_.current() > 0.0) {
 435     const int buffer_capable_area = base::saturated_cast<int>(
 436         current_area / buffer_pool_utilization_.current());
 437     if (buffer_capable_area < increased_area) {
 438       start_time_of_underutilization_ = base::TimeTicks();
 439       return -1;  // Buffer pool is not under-utilized.
 440     }
 441   }
 442
 443   // Determine whether the consumer could handle an increase in area.
 444   if (HasSufficientRecentFeedback(estimated_capable_area_, analyze_time)) {
 445     if (estimated_capable_area_.current() < increased_area) {
 446       start_time_of_underutilization_ = base::TimeTicks();
 447       return -1;  // Consumer is not under-utilized.
 448     }
 449   } else if (estimated_capable_area_.update_time() ==
 450              estimated_capable_area_.reset_time()) {
 451     // The consumer does not provide any feedback.  In this case, the consumer's
 452     // capability isn't a consideration.
 453   } else {
 454     // Consumer is providing feedback, but hasn't reported it recently.  Just in
 455     // case it's stalled, don't make things worse by increasing the capture
 456     // area.
 457     start_time_of_underutilization_ = base::TimeTicks();
 458     return -1;
 459   }
 460
 461   // At this point, the system is currently under-utilized.  Reset the start
 462   // time if the system was not under-utilized when the last analysis was made.
 463   if (start_time_of_underutilization_.is_null())
 464     start_time_of_underutilization_ = analyze_time;
 465
 466   // While content is animating, require a "proving period" of contiguous
 467   // under-utilization before increasing the capture area.  This will mitigate
 468   // the risk of causing frames to be dropped when increasing the load.  If
 469   // content is not animating, be aggressive about increasing the capture area,
 470   // to improve the quality of non-animating content (where frame drops are not
 471   // much of a concern).
 472   if ((analyze_time - last_time_animation_was_detected_).InMicroseconds() <
 473       kDebouncingPeriodForAnimatedContentMicros) {
 474     if ((analyze_time - start_time_of_underutilization_).InMicroseconds() <
 475         kProvingPeriodForAnimatedContentMicros) {
 476       // Content is animating and the system has not been contiguously
 477       // under-utilizated for long enough.
 478       return -1;
 479     }
 480   }
 481
 482   VLOG(2) << "Proposing a "
 483           << (100.0 * (increased_area - current_area) / current_area)
 484           << "% increase in capture area.  :-)";
 485
 486   return increased_area;
 487 }
 488
 489 }  // namespace media