content/browser/media/capture/web_contents_video_capture_device.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Implementation notes: This needs to work on a variety of hardware
   6 // configurations where the speed of the CPU and GPU greatly affect overall
   7 // performance. Spanning several threads, the process of capturing has been
   8 // split up into four conceptual stages:
   9 //
  10 //   1. Reserve Buffer: Before a frame can be captured, a slot in the client's
  11 //      shared-memory IPC buffer is reserved. There are only a few of these;
  12 //      when they run out, it indicates that the downstream client -- likely a
  13 //      video encoder -- is the performance bottleneck, and that the rate of
  14 //      frame capture should be throttled back.
  15 //
  16 //   2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing
  17 //      store. This is initiated on the UI BrowserThread, and often occurs
  18 //      asynchronously. Where supported, the GPU scales and color converts
  19 //      frames to our desired size, and the readback happens directly into the
  20 //      shared-memory buffer. But this is not always possible, particularly when
  21 //      accelerated compositing is disabled.
  22 //
  23 //   3. Render (if needed): If the web contents cannot be captured directly into
  24 //      our target size and color format, scaling and colorspace conversion must
  25 //      be done on the CPU. A dedicated thread is used for this operation, to
  26 //      avoid blocking the UI thread. The Render stage always reads from a
  27 //      bitmap returned by Capture, and writes into the reserved slot in the
  28 //      shared-memory buffer.
  29 //
  30 //   4. Deliver: The rendered video frame is returned to the client (which
  31 //      implements the VideoCaptureDevice::Client interface). Because all
  32 //      paths have written the frame into the IPC buffer, this step should
  33 //      never need to do an additional copy of the pixel data.
  34 //
  35 // In the best-performing case, the Render step is bypassed: Capture produces
  36 // ready-to-Deliver frames. But when accelerated readback is not possible, the
  37 // system is designed so that Capture and Render may run concurrently. A timing
  38 // diagram helps illustrate this point (@30 FPS):
  39 //
  40 //    Time: 0ms                 33ms                 66ms                 99ms
  41 // thread1: |-Capture-f1------v |-Capture-f2------v  |-Capture-f3----v    |-Capt
  42 // thread2:                   |-Render-f1-----v   |-Render-f2-----v  |-Render-f3
  43 //
  44 // In the above example, both capturing and rendering *each* take almost the
  45 // full 33 ms available between frames, yet we see that the required throughput
  46 // is obtained.
  47 //
  48 // Turning on verbose logging will cause the effective frame rate to be logged
  49 // at 5-second intervals.
  50
  51 #include "content/browser/media/capture/web_contents_video_capture_device.h"
  52
  53 #include "base/basictypes.h"
  54 #include "base/bind.h"
  55 #include "base/callback_helpers.h"
  56 #include "base/location.h"
  57 #include "base/logging.h"
  58 #include "base/memory/scoped_ptr.h"
  59 #include "base/memory/weak_ptr.h"
  60 #include "base/metrics/histogram.h"
  61 #include "base/sequenced_task_runner.h"
  62 #include "base/single_thread_task_runner.h"
  63 #include "base/threading/thread.h"
  64 #include "base/threading/thread_checker.h"
  65 #include "base/time/time.h"
  66 #include "content/browser/media/capture/web_contents_capture_util.h"
  67 #include "content/browser/media/capture/web_contents_tracker.h"
  68 #include "content/browser/renderer_host/render_widget_host_impl.h"
  69 #include "content/browser/renderer_host/render_widget_host_view_base.h"
  70 #include "content/public/browser/browser_thread.h"
  71 #include "content/public/browser/render_process_host.h"
  72 #include "content/public/browser/render_widget_host_view.h"
  73 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
  74 #include "content/public/browser/web_contents.h"
  75 #include "media/base/video_capture_types.h"
  76 #include "media/base/video_util.h"
  77 #include "media/capture/content/screen_capture_device_core.h"
  78 #include "media/capture/content/thread_safe_capture_oracle.h"
  79 #include "media/capture/content/video_capture_oracle.h"
  80 #include "skia/ext/image_operations.h"
  81 #include "third_party/skia/include/core/SkBitmap.h"
  82 #include "third_party/skia/include/core/SkColor.h"
  83 #include "ui/base/layout.h"
  84 #include "ui/gfx/geometry/dip_util.h"
  85 #include "ui/gfx/geometry/size_conversions.h"
  86
  87 namespace content {
  88
  89 namespace {
  90
  91 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
  92                           const base::Closure& callback) {
  93   render_thread.reset();
  94
  95   // After thread join call the callback on UI thread.
  96   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback);
  97 }
  98
  99 // Responsible for logging the effective frame rate.
 100 class VideoFrameDeliveryLog {
 101  public:
 102   VideoFrameDeliveryLog();
 103
 104   // Report that the frame posted with |frame_time| has been delivered.
 105   void ChronicleFrameDelivery(base::TimeTicks frame_time);
 106
 107  private:
 108   // The following keep track of and log the effective frame rate whenever
 109   // verbose logging is turned on.
 110   base::TimeTicks last_frame_rate_log_time_;
 111   int count_frames_rendered_;
 112
 113   DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
 114 };
 115
 116 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
 117 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
 118 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
 119  public:
 120   FrameSubscriber(media::VideoCaptureOracle::Event event_type,
 121                   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle,
 122                   VideoFrameDeliveryLog* delivery_log)
 123       : event_type_(event_type),
 124         oracle_proxy_(oracle),
 125         delivery_log_(delivery_log) {}
 126
 127   bool ShouldCaptureFrame(
 128       const gfx::Rect& damage_rect,
 129       base::TimeTicks present_time,
 130       scoped_refptr<media::VideoFrame>* storage,
 131       RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
 132           deliver_frame_cb) override;
 133
 134  private:
 135   const media::VideoCaptureOracle::Event event_type_;
 136   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
 137   VideoFrameDeliveryLog* const delivery_log_;
 138 };
 139
 140 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
 141 // whose content is updating, a subscriber that is deciding which of these
 142 // updates to capture (and where to deliver them to), and a callback that
 143 // knows how to do the capture and prepare the result for delivery.
 144 //
 145 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
 146 // the RenderWidgetHostView, to process compositor updates, and (b) running a
 147 // timer to possibly initiate forced, non-event-driven captures needed by
 148 // downstream consumers that require frame repeats of unchanged content.
 149 //
 150 // All of this happens on the UI thread, although the
 151 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates
 152 // autonomously on some other thread.
 153 class ContentCaptureSubscription {
 154  public:
 155   typedef base::Callback<
 156       void(const base::TimeTicks&,
 157            const scoped_refptr<media::VideoFrame>&,
 158            const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>
 159       CaptureCallback;
 160
 161   // Create a subscription. Whenever a manual capture is required, the
 162   // subscription will invoke |capture_callback| on the UI thread to do the
 163   // work.
 164   ContentCaptureSubscription(
 165       const RenderWidgetHost& source,
 166       const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 167       const CaptureCallback& capture_callback);
 168   ~ContentCaptureSubscription();
 169
 170  private:
 171   void OnTimer();
 172
 173   // Maintain a weak reference to the RenderWidgetHost (via its routing ID),
 174   // since the instance could be destroyed externally during the lifetime of
 175   // |this|.
 176   const int render_process_id_;
 177   const int render_widget_id_;
 178
 179   VideoFrameDeliveryLog delivery_log_;
 180   FrameSubscriber timer_subscriber_;
 181   CaptureCallback capture_callback_;
 182   base::Timer timer_;
 183
 184   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 185 };
 186
 187 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
 188 // invoke |done_cb| to indicate success or failure. |input| is expected to be
 189 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
 190 // Scaling and letterboxing will be done as needed.
 191 //
 192 // This software implementation should be used only when GPU acceleration of
 193 // these activities is not possible. This operation may be expensive (tens to
 194 // hundreds of milliseconds), so the caller should ensure that it runs on a
 195 // thread where such a pause would cause UI jank.
 196 void RenderVideoFrame(const SkBitmap& input,
 197                       const scoped_refptr<media::VideoFrame>& output,
 198                       const base::Callback<void(bool)>& done_cb);
 199
 200 // Renews capture subscriptions based on feedback from WebContentsTracker, and
 201 // also executes copying of the backing store on the UI BrowserThread.
 202 class WebContentsCaptureMachine : public media::VideoCaptureMachine {
 203  public:
 204   WebContentsCaptureMachine(int render_process_id,
 205                             int main_render_frame_id,
 206                             bool enable_auto_throttling);
 207   ~WebContentsCaptureMachine() override;
 208
 209   // VideoCaptureMachine overrides.
 210   void Start(const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 211              const media::VideoCaptureParams& params,
 212              const base::Callback<void(bool)> callback) override;
 213   void Stop(const base::Closure& callback) override;
 214   bool IsAutoThrottlingEnabled() const override {
 215     return auto_throttling_enabled_;
 216   }
 217
 218   // Starts a copy from the backing store or the composited surface. Must be run
 219   // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation
 220   // completes. The copy will occur to |target|.
 221   //
 222   // This may be used as a ContentCaptureSubscription::CaptureCallback.
 223   void Capture(const base::TimeTicks& start_time,
 224                const scoped_refptr<media::VideoFrame>& target,
 225                const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 226                    deliver_frame_cb);
 227
 228  private:
 229   bool InternalStart(
 230       const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 231       const media::VideoCaptureParams& params);
 232   void InternalStop(const base::Closure& callback);
 233   bool IsStarted() const;
 234
 235   // Computes the preferred size of the target RenderWidget for optimal capture.
 236   gfx::Size ComputeOptimalViewSize() const;
 237
 238   // Response callback for RenderWidgetHost::CopyFromBackingStore().
 239   void DidCopyFromBackingStore(
 240       const base::TimeTicks& start_time,
 241       const scoped_refptr<media::VideoFrame>& target,
 242       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 243           deliver_frame_cb,
 244       const SkBitmap& bitmap,
 245       ReadbackResponse response);
 246
 247   // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
 248   void DidCopyFromCompositingSurfaceToVideoFrame(
 249       const base::TimeTicks& start_time,
 250       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 251           deliver_frame_cb,
 252       bool success);
 253
 254   // Remove the old subscription, and attempt to start a new one if |had_target|
 255   // is true.
 256   void RenewFrameSubscription(bool had_target);
 257
 258   // Called whenever the render widget is resized.
 259   void UpdateCaptureSize();
 260
 261   // Parameters saved in constructor.
 262   const int initial_render_process_id_;
 263   const int initial_main_render_frame_id_;
 264
 265   // Tracks events and calls back to RenewFrameSubscription() to maintain
 266   // capture on the correct RenderWidgetHost.
 267   const scoped_refptr<WebContentsTracker> tracker_;
 268
 269   // Set to false to prevent the capture size from automatically adjusting in
 270   // response to end-to-end utilization.  This is enabled via the throttling
 271   // option in the WebContentsVideoCaptureDevice device ID.
 272   const bool auto_throttling_enabled_;
 273
 274   // A dedicated worker thread on which SkBitmap->VideoFrame conversion will
 275   // occur. Only used when this activity cannot be done on the GPU.
 276   scoped_ptr<base::Thread> render_thread_;
 277
 278   // Makes all the decisions about which frames to copy, and how.
 279   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
 280
 281   // Video capture parameters that this machine is started with.
 282   media::VideoCaptureParams capture_params_;
 283
 284   // Last known RenderView size.
 285   gfx::Size last_view_size_;
 286
 287   // Responsible for forwarding events from the active RenderWidgetHost to the
 288   // oracle, and initiating captures accordingly.
 289   scoped_ptr<ContentCaptureSubscription> subscription_;
 290
 291   // Weak pointer factory used to invalidate callbacks.
 292   // NOTE: Weak pointers must be invalidated before all other member variables.
 293   base::WeakPtrFactory<WebContentsCaptureMachine> weak_ptr_factory_;
 294
 295   DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine);
 296 };
 297
 298 bool FrameSubscriber::ShouldCaptureFrame(
 299     const gfx::Rect& damage_rect,
 300     base::TimeTicks present_time,
 301     scoped_refptr<media::VideoFrame>* storage,
 302     DeliverFrameCallback* deliver_frame_cb) {
 303   TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame",
 304                "instance", this);
 305
 306   media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
 307   bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture(
 308       event_type_, damage_rect, present_time, storage, &capture_frame_cb);
 309
 310   if (!capture_frame_cb.is_null())
 311     *deliver_frame_cb = base::Bind(capture_frame_cb, *storage);
 312   if (oracle_decision)
 313     delivery_log_->ChronicleFrameDelivery(present_time);
 314   return oracle_decision;
 315 }
 316
 317 ContentCaptureSubscription::ContentCaptureSubscription(
 318     const RenderWidgetHost& source,
 319     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 320     const CaptureCallback& capture_callback)
 321     : render_process_id_(source.GetProcess()->GetID()),
 322       render_widget_id_(source.GetRoutingID()),
 323       delivery_log_(),
 324       timer_subscriber_(media::VideoCaptureOracle::kTimerPoll, oracle_proxy,
 325                         &delivery_log_),
 326       capture_callback_(capture_callback),
 327       timer_(true, true) {
 328   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 329
 330   RenderWidgetHostView* const view = source.GetView();
 331
 332   // Subscribe to compositor updates. These will be serviced directly by the
 333   // oracle.
 334   if (view) {
 335     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
 336         new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate,
 337             oracle_proxy, &delivery_log_));
 338     view->BeginFrameSubscription(subscriber.Pass());
 339   }
 340
 341   // Subscribe to timer events. This instance will service these as well.
 342   timer_.Start(FROM_HERE,
 343                std::max(oracle_proxy->min_capture_period(),
 344                         base::TimeDelta::FromMilliseconds(media
 345                             ::VideoCaptureOracle::kMinTimerPollPeriodMillis)),
 346                base::Bind(&ContentCaptureSubscription::OnTimer,
 347                           base::Unretained(this)));
 348 }
 349
 350 ContentCaptureSubscription::~ContentCaptureSubscription() {
 351   // If the BrowserThreads have been torn down, then the browser is in the final
 352   // stages of exiting and it is dangerous to take any further action.  We must
 353   // return early.  http://crbug.com/396413
 354   if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI))
 355     return;
 356
 357   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 358   RenderWidgetHost* const source =
 359       RenderWidgetHost::FromID(render_process_id_, render_widget_id_);
 360   RenderWidgetHostView* const view = source ? source->GetView() : NULL;
 361   if (view)
 362     view->EndFrameSubscription();
 363 }
 364
 365 void ContentCaptureSubscription::OnTimer() {
 366   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 367   TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnTimer");
 368
 369   scoped_refptr<media::VideoFrame> frame;
 370   RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
 371
 372   const base::TimeTicks start_time = base::TimeTicks::Now();
 373   if (timer_subscriber_.ShouldCaptureFrame(gfx::Rect(),
 374                                            start_time,
 375                                            &frame,
 376                                            &deliver_frame_cb)) {
 377     capture_callback_.Run(start_time, frame, deliver_frame_cb);
 378   }
 379 }
 380
 381 void RenderVideoFrame(const SkBitmap& input,
 382                       const scoped_refptr<media::VideoFrame>& output,
 383                       const base::Callback<void(bool)>& done_cb) {
 384   base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
 385
 386   SkAutoLockPixels locker(input);
 387
 388   // Sanity-check the captured bitmap.
 389   if (input.empty() ||
 390       !input.readyToDraw() ||
 391       input.colorType() != kN32_SkColorType ||
 392       input.width() < 2 || input.height() < 2) {
 393     DVLOG(1) << "input unacceptable (size="
 394              << input.getSize()
 395              << ", ready=" << input.readyToDraw()
 396              << ", colorType=" << input.colorType() << ')';
 397     return;
 398   }
 399
 400   // Sanity-check the output buffer.
 401   if (output->format() != media::PIXEL_FORMAT_I420) {
 402     NOTREACHED();
 403     return;
 404   }
 405
 406   // Calculate the width and height of the content region in the |output|, based
 407   // on the aspect ratio of |input|.
 408   const gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
 409       output->visible_rect(), gfx::Size(input.width(), input.height()));
 410
 411   // Scale the bitmap to the required size, if necessary.
 412   SkBitmap scaled_bitmap;
 413   if (input.width() != region_in_frame.width() ||
 414       input.height() != region_in_frame.height()) {
 415
 416     skia::ImageOperations::ResizeMethod method;
 417     if (input.width() < region_in_frame.width() ||
 418         input.height() < region_in_frame.height()) {
 419       // Avoid box filtering when magnifying, because it's actually
 420       // nearest-neighbor.
 421       method = skia::ImageOperations::RESIZE_HAMMING1;
 422     } else {
 423       method = skia::ImageOperations::RESIZE_BOX;
 424     }
 425
 426     TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture",
 427                                  "Capture", output.get(), "Scale");
 428     scaled_bitmap = skia::ImageOperations::Resize(input, method,
 429                                                   region_in_frame.width(),
 430                                                   region_in_frame.height());
 431   } else {
 432     scaled_bitmap = input;
 433   }
 434
 435   TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output.get(), "YUV");
 436   {
 437     // Align to 2x2 pixel boundaries, as required by
 438     // media::CopyRGBToVideoFrame().
 439     const gfx::Rect region_in_yv12_frame(region_in_frame.x() & ~1,
 440                                          region_in_frame.y() & ~1,
 441                                          region_in_frame.width() & ~1,
 442                                          region_in_frame.height() & ~1);
 443     if (region_in_yv12_frame.IsEmpty())
 444       return;
 445
 446     SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap);
 447     media::CopyRGBToVideoFrame(
 448         reinterpret_cast<uint8*>(scaled_bitmap.getPixels()),
 449         scaled_bitmap.rowBytes(),
 450         region_in_yv12_frame,
 451         output.get());
 452   }
 453
 454   // The result is now ready.
 455   ignore_result(failure_handler.Release());
 456   done_cb.Run(true);
 457 }
 458
 459 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
 460     : last_frame_rate_log_time_(),
 461       count_frames_rendered_(0) {
 462 }
 463
 464 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time) {
 465   // Log frame rate, if verbose logging is turned on.
 466   static const base::TimeDelta kFrameRateLogInterval =
 467       base::TimeDelta::FromSeconds(10);
 468   if (last_frame_rate_log_time_.is_null()) {
 469     last_frame_rate_log_time_ = frame_time;
 470     count_frames_rendered_ = 0;
 471   } else {
 472     ++count_frames_rendered_;
 473     const base::TimeDelta elapsed = frame_time - last_frame_rate_log_time_;
 474     if (elapsed >= kFrameRateLogInterval) {
 475       const double measured_fps =
 476           count_frames_rendered_ / elapsed.InSecondsF();
 477       UMA_HISTOGRAM_COUNTS(
 478           "TabCapture.FrameRate",
 479           static_cast<int>(measured_fps));
 480       VLOG(1) << "Current measured frame rate for "
 481               << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS.";
 482       last_frame_rate_log_time_ = frame_time;
 483       count_frames_rendered_ = 0;
 484     }
 485   }
 486 }
 487
 488 WebContentsCaptureMachine::WebContentsCaptureMachine(
 489     int render_process_id,
 490     int main_render_frame_id,
 491     bool enable_auto_throttling)
 492     : initial_render_process_id_(render_process_id),
 493       initial_main_render_frame_id_(main_render_frame_id),
 494       tracker_(new WebContentsTracker(true)),
 495       auto_throttling_enabled_(enable_auto_throttling),
 496       weak_ptr_factory_(this) {
 497   DVLOG(1) << "Created WebContentsCaptureMachine for "
 498            << render_process_id << ':' << main_render_frame_id
 499            << (auto_throttling_enabled_ ? " with auto-throttling enabled" : "");
 500 }
 501
 502 WebContentsCaptureMachine::~WebContentsCaptureMachine() {}
 503
 504 bool WebContentsCaptureMachine::IsStarted() const {
 505   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 506   return weak_ptr_factory_.HasWeakPtrs();
 507 }
 508
 509 void WebContentsCaptureMachine::Start(
 510     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 511     const media::VideoCaptureParams& params,
 512     const base::Callback<void(bool)> callback) {
 513   // Starts the capture machine asynchronously.
 514   BrowserThread::PostTaskAndReplyWithResult(
 515       BrowserThread::UI,
 516       FROM_HERE,
 517       base::Bind(&WebContentsCaptureMachine::InternalStart,
 518                  base::Unretained(this),
 519                  oracle_proxy,
 520                  params),
 521       callback);
 522 }
 523
 524 bool WebContentsCaptureMachine::InternalStart(
 525     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 526     const media::VideoCaptureParams& params) {
 527   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 528   DCHECK(!IsStarted());
 529
 530   DCHECK(oracle_proxy.get());
 531   oracle_proxy_ = oracle_proxy;
 532   capture_params_ = params;
 533
 534   render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread"));
 535   if (!render_thread_->Start()) {
 536     DVLOG(1) << "Failed to spawn render thread.";
 537     render_thread_.reset();
 538     return false;
 539   }
 540
 541   // Note: Creation of the first WeakPtr in the following statement will cause
 542   // IsStarted() to return true from now on.
 543   tracker_->SetResizeChangeCallback(
 544       base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize,
 545                  weak_ptr_factory_.GetWeakPtr()));
 546   tracker_->Start(initial_render_process_id_, initial_main_render_frame_id_,
 547                   base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription,
 548                              weak_ptr_factory_.GetWeakPtr()));
 549
 550   return true;
 551 }
 552
 553 void WebContentsCaptureMachine::Stop(const base::Closure& callback) {
 554   // Stops the capture machine asynchronously.
 555   BrowserThread::PostTask(
 556       BrowserThread::UI, FROM_HERE, base::Bind(
 557           &WebContentsCaptureMachine::InternalStop,
 558           base::Unretained(this),
 559           callback));
 560 }
 561
 562 void WebContentsCaptureMachine::InternalStop(const base::Closure& callback) {
 563   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 564
 565   if (!IsStarted()) {
 566     callback.Run();
 567     return;
 568   }
 569
 570   // The following cancels any outstanding callbacks and causes IsStarted() to
 571   // return false from here onward.
 572   weak_ptr_factory_.InvalidateWeakPtrs();
 573
 574   // Note: RenewFrameSubscription() must be called before stopping |tracker_| so
 575   // the web_contents() can be notified that the capturing is ending.
 576   RenewFrameSubscription(false);
 577   tracker_->Stop();
 578
 579   // The render thread cannot be stopped on the UI thread, so post a message
 580   // to the thread pool used for blocking operations.
 581   if (render_thread_.get()) {
 582     BrowserThread::PostBlockingPoolTask(
 583         FROM_HERE,
 584         base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_),
 585                    callback));
 586   }
 587 }
 588
 589 void WebContentsCaptureMachine::Capture(
 590     const base::TimeTicks& start_time,
 591     const scoped_refptr<media::VideoFrame>& target,
 592     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 593         deliver_frame_cb) {
 594   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 595
 596   RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost();
 597   RenderWidgetHostViewBase* view =
 598       rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL;
 599   if (!view) {
 600     deliver_frame_cb.Run(base::TimeTicks(), false);
 601     return;
 602   }
 603
 604   gfx::Size view_size = view->GetViewBounds().size();
 605   if (view_size != last_view_size_) {
 606     last_view_size_ = view_size;
 607
 608     // Measure the number of kilopixels.
 609     UMA_HISTOGRAM_COUNTS_10000(
 610         "TabCapture.ViewChangeKiloPixels",
 611         view_size.width() * view_size.height() / 1024);
 612   }
 613
 614   if (view->CanCopyToVideoFrame()) {
 615     view->CopyFromCompositingSurfaceToVideoFrame(
 616         gfx::Rect(view_size),
 617         target,
 618         base::Bind(&WebContentsCaptureMachine::
 619                         DidCopyFromCompositingSurfaceToVideoFrame,
 620                    weak_ptr_factory_.GetWeakPtr(),
 621                    start_time, deliver_frame_cb));
 622   } else {
 623     const gfx::Size fitted_size = view_size.IsEmpty() ? gfx::Size() :
 624         media::ComputeLetterboxRegion(target->visible_rect(), view_size).size();
 625     rwh->CopyFromBackingStore(
 626         gfx::Rect(),
 627         fitted_size,  // Size here is a request not always honored.
 628         base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore,
 629                    weak_ptr_factory_.GetWeakPtr(),
 630                    start_time,
 631                    target,
 632                    deliver_frame_cb),
 633         kN32_SkColorType);
 634   }
 635 }
 636
 637 gfx::Size WebContentsCaptureMachine::ComputeOptimalViewSize() const {
 638   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 639
 640   // TODO(miu): Propagate capture frame size changes as new "preferred size"
 641   // updates, rather than just using the max frame size.
 642   // http://crbug.com/350491
 643   gfx::Size optimal_size = oracle_proxy_->max_frame_size();
 644
 645   switch (capture_params_.resolution_change_policy) {
 646     case media::RESOLUTION_POLICY_FIXED_RESOLUTION:
 647       break;
 648     case media::RESOLUTION_POLICY_FIXED_ASPECT_RATIO:
 649     case media::RESOLUTION_POLICY_ANY_WITHIN_LIMIT: {
 650       // If the max frame size is close to a common video aspect ratio, compute
 651       // a standard resolution for that aspect ratio.  For example, given
 652       // 1365x768, which is very close to 16:9, the optimal size would be
 653       // 1280x720.  The purpose of this logic is to prevent scaling quality
 654       // issues caused by "one pixel stretching" and/or odd-to-even dimension
 655       // scaling, and to improve the performance of consumers of the captured
 656       // video.
 657       const auto HasIntendedAspectRatio =
 658           [](const gfx::Size& size, int width_units, int height_units) {
 659         const int a = height_units * size.width();
 660         const int b = width_units * size.height();
 661         const int percentage_diff = 100 * std::abs((a - b)) / b;
 662         return percentage_diff <= 1;  // Effectively, anything strictly <2%.
 663       };
 664       const auto RoundToExactAspectRatio =
 665           [](const gfx::Size& size, int width_step, int height_step) {
 666         const int adjusted_height =
 667             std::max(size.height() - (size.height() % height_step),
 668                      height_step);
 669         DCHECK_EQ((adjusted_height * width_step) % height_step, 0);
 670         return gfx::Size(adjusted_height * width_step / height_step,
 671                          adjusted_height);
 672       };
 673       if (HasIntendedAspectRatio(optimal_size, 16, 9))
 674         optimal_size = RoundToExactAspectRatio(optimal_size, 160, 90);
 675       else if (HasIntendedAspectRatio(optimal_size, 4, 3))
 676         optimal_size = RoundToExactAspectRatio(optimal_size, 64, 48);
 677       // Else, do not make an adjustment.
 678       break;
 679     }
 680   }
 681
 682   // If the ratio between physical and logical pixels is greater than 1:1,
 683   // shrink |optimal_size| by that amount.  Then, when external code resizes the
 684   // render widget to the "preferred size," the widget will be physically
 685   // rendered at the exact capture size, thereby eliminating unnecessary scaling
 686   // operations in the graphics pipeline.
 687   RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost();
 688   RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : NULL;
 689   if (rwhv) {
 690     const gfx::NativeView view = rwhv->GetNativeView();
 691     const float scale = ui::GetScaleFactorForNativeView(view);
 692     if (scale > 1.0f) {
 693       const gfx::Size shrunk_size(
 694           gfx::ToFlooredSize(gfx::ScaleSize(optimal_size, 1.0f / scale)));
 695       if (shrunk_size.width() > 0 && shrunk_size.height() > 0)
 696         optimal_size = shrunk_size;
 697     }
 698   }
 699
 700   VLOG(1) << "Computed optimal target size: " << optimal_size.ToString();
 701   return optimal_size;
 702 }
 703
 704 void WebContentsCaptureMachine::DidCopyFromBackingStore(
 705     const base::TimeTicks& start_time,
 706     const scoped_refptr<media::VideoFrame>& target,
 707     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 708         deliver_frame_cb,
 709     const SkBitmap& bitmap,
 710     ReadbackResponse response) {
 711   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 712
 713   base::TimeTicks now = base::TimeTicks::Now();
 714   DCHECK(render_thread_.get());
 715   if (response == READBACK_SUCCESS) {
 716     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time);
 717     TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(),
 718                                  "Render");
 719     render_thread_->task_runner()->PostTask(
 720         FROM_HERE, base::Bind(&RenderVideoFrame, bitmap, target,
 721                               base::Bind(deliver_frame_cb, start_time)));
 722   } else {
 723     // Capture can fail due to transient issues, so just skip this frame.
 724     DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
 725     deliver_frame_cb.Run(start_time, false);
 726   }
 727 }
 728
 729 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
 730     const base::TimeTicks& start_time,
 731     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 732         deliver_frame_cb,
 733     bool success) {
 734   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 735   base::TimeTicks now = base::TimeTicks::Now();
 736
 737   if (success) {
 738     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time);
 739   } else {
 740     // Capture can fail due to transient issues, so just skip this frame.
 741     DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
 742   }
 743   deliver_frame_cb.Run(start_time, success);
 744 }
 745
 746 void WebContentsCaptureMachine::RenewFrameSubscription(bool had_target) {
 747   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 748
 749   RenderWidgetHost* const rwh =
 750       had_target ? tracker_->GetTargetRenderWidgetHost() : nullptr;
 751
 752   // Always destroy the old subscription before creating a new one.
 753   const bool had_subscription = !!subscription_;
 754   subscription_.reset();
 755
 756   DVLOG(1) << "Renewing frame subscription to RWH@" << rwh
 757            << ", had_subscription=" << had_subscription;
 758
 759   if (!rwh) {
 760     if (had_subscription && tracker_->web_contents())
 761       tracker_->web_contents()->DecrementCapturerCount();
 762     if (IsStarted()) {
 763       // Tracking of WebContents and/or its main frame has failed before Stop()
 764       // was called, so report this as an error:
 765       oracle_proxy_->ReportError("WebContents and/or main frame are gone.");
 766     }
 767     return;
 768   }
 769
 770   if (!had_subscription && tracker_->web_contents())
 771     tracker_->web_contents()->IncrementCapturerCount(ComputeOptimalViewSize());
 772
 773   subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_,
 774       base::Bind(&WebContentsCaptureMachine::Capture,
 775                  weak_ptr_factory_.GetWeakPtr())));
 776 }
 777
 778 void WebContentsCaptureMachine::UpdateCaptureSize() {
 779   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 780
 781   if (!oracle_proxy_)
 782     return;
 783   RenderWidgetHost* const rwh = tracker_->GetTargetRenderWidgetHost();
 784   RenderWidgetHostView* const view = rwh ? rwh->GetView() : nullptr;
 785   if (!view)
 786     return;
 787
 788   // Convert the view's size from the DIP coordinate space to the pixel
 789   // coordinate space.  When the view is being rendered on a high-DPI display,
 790   // this allows the high-resolution image detail to propagate through to the
 791   // captured video.
 792   const gfx::Size view_size = view->GetViewBounds().size();
 793   const gfx::Size physical_size = gfx::ConvertSizeToPixel(
 794       ui::GetScaleFactorForNativeView(view->GetNativeView()), view_size);
 795   VLOG(1) << "Computed physical capture size (" << physical_size.ToString()
 796           << ") from view size (" << view_size.ToString() << ").";
 797
 798   oracle_proxy_->UpdateCaptureSize(physical_size);
 799 }
 800
 801 }  // namespace
 802
 803 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice(
 804     int render_process_id,
 805     int main_render_frame_id,
 806     bool enable_auto_throttling)
 807     : core_(new media::ScreenCaptureDeviceCore(
 808           scoped_ptr<media::VideoCaptureMachine>(new WebContentsCaptureMachine(
 809               render_process_id,
 810               main_render_frame_id,
 811               enable_auto_throttling)))) {}
 812
 813 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() {
 814   DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying.";
 815 }
 816
 817 // static
 818 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create(
 819     const std::string& device_id) {
 820   // Parse device_id into render_process_id and main_render_frame_id.
 821   int render_process_id = -1;
 822   int main_render_frame_id = -1;
 823   if (!WebContentsCaptureUtil::ExtractTabCaptureTarget(
 824            device_id, &render_process_id, &main_render_frame_id)) {
 825     return NULL;
 826   }
 827
 828   return new WebContentsVideoCaptureDevice(
 829       render_process_id,
 830       main_render_frame_id,
 831       WebContentsCaptureUtil::IsAutoThrottlingOptionSet(device_id));
 832 }
 833
 834 void WebContentsVideoCaptureDevice::AllocateAndStart(
 835     const media::VideoCaptureParams& params,
 836     scoped_ptr<Client> client) {
 837   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
 838   core_->AllocateAndStart(params, client.Pass());
 839 }
 840
 841 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
 842   core_->StopAndDeAllocate();
 843 }
 844
 845 }  // namespace content