content/browser/media/capture/web_contents_video_capture_device.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Implementation notes: This needs to work on a variety of hardware
   6 // configurations where the speed of the CPU and GPU greatly affect overall
   7 // performance. Spanning several threads, the process of capturing has been
   8 // split up into four conceptual stages:
   9 //
  10 //   1. Reserve Buffer: Before a frame can be captured, a slot in the client's
  11 //      shared-memory IPC buffer is reserved. There are only a few of these;
  12 //      when they run out, it indicates that the downstream client -- likely a
  13 //      video encoder -- is the performance bottleneck, and that the rate of
  14 //      frame capture should be throttled back.
  15 //
  16 //   2. Capture: A bitmap is snapshotted/copied from the RenderView's backing
  17 //      store. This is initiated on the UI BrowserThread, and often occurs
  18 //      asynchronously. Where supported, the GPU scales and color converts
  19 //      frames to our desired size, and the readback happens directly into the
  20 //      shared-memory buffer. But this is not always possible, particularly when
  21 //      accelerated compositing is disabled.
  22 //
  23 //   3. Render (if needed): If the web contents cannot be captured directly into
  24 //      our target size and color format, scaling and colorspace conversion must
  25 //      be done on the CPU. A dedicated thread is used for this operation, to
  26 //      avoid blocking the UI thread. The Render stage always reads from a
  27 //      bitmap returned by Capture, and writes into the reserved slot in the
  28 //      shared-memory buffer.
  29 //
  30 //   4. Deliver: The rendered video frame is returned to the client (which
  31 //      implements the VideoCaptureDevice::Client interface). Because all
  32 //      paths have written the frame into the IPC buffer, this step should
  33 //      never need to do an additional copy of the pixel data.
  34 //
  35 // In the best-performing case, the Render step is bypassed: Capture produces
  36 // ready-to-Deliver frames. But when accelerated readback is not possible, the
  37 // system is designed so that Capture and Render may run concurrently. A timing
  38 // diagram helps illustrate this point (@30 FPS):
  39 //
  40 //    Time: 0ms                 33ms                 66ms                 99ms
  41 // thread1: |-Capture-f1------v |-Capture-f2------v  |-Capture-f3----v    |-Capt
  42 // thread2:                   |-Render-f1-----v   |-Render-f2-----v  |-Render-f3
  43 //
  44 // In the above example, both capturing and rendering *each* take almost the
  45 // full 33 ms available between frames, yet we see that the required throughput
  46 // is obtained.
  47 //
  48 // Turning on verbose logging will cause the effective frame rate to be logged
  49 // at 5-second intervals.
  50
  51 #include "content/browser/media/capture/web_contents_video_capture_device.h"
  52
  53 #include "base/basictypes.h"
  54 #include "base/bind.h"
  55 #include "base/callback_helpers.h"
  56 #include "base/logging.h"
  57 #include "base/memory/scoped_ptr.h"
  58 #include "base/memory/weak_ptr.h"
  59 #include "base/message_loop/message_loop_proxy.h"
  60 #include "base/metrics/histogram.h"
  61 #include "base/sequenced_task_runner.h"
  62 #include "base/threading/thread.h"
  63 #include "base/threading/thread_checker.h"
  64 #include "base/time/time.h"
  65 #include "content/browser/media/capture/content_video_capture_device_core.h"
  66 #include "content/browser/media/capture/video_capture_oracle.h"
  67 #include "content/browser/media/capture/web_contents_capture_util.h"
  68 #include "content/browser/renderer_host/render_widget_host_impl.h"
  69 #include "content/browser/renderer_host/render_widget_host_view_base.h"
  70 #include "content/browser/web_contents/web_contents_impl.h"
  71 #include "content/public/browser/browser_thread.h"
  72 #include "content/public/browser/notification_source.h"
  73 #include "content/public/browser/notification_types.h"
  74 #include "content/public/browser/render_view_host.h"
  75 #include "content/public/browser/render_widget_host_view.h"
  76 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
  77 #include "content/public/browser/web_contents_observer.h"
  78 #include "media/base/video_util.h"
  79 #include "media/video/capture/video_capture_types.h"
  80 #include "skia/ext/image_operations.h"
  81 #include "third_party/skia/include/core/SkBitmap.h"
  82 #include "third_party/skia/include/core/SkColor.h"
  83 #include "ui/gfx/display.h"
  84 #include "ui/gfx/geometry/size.h"
  85 #include "ui/gfx/geometry/size_conversions.h"
  86 #include "ui/gfx/screen.h"
  87
  88 namespace content {
  89
  90 namespace {
  91
  92 // Compute a letterbox region, aligned to even coordinates.
  93 gfx::Rect ComputeYV12LetterboxRegion(const gfx::Size& frame_size,
  94                                      const gfx::Size& content_size) {
  95
  96   gfx::Rect result = media::ComputeLetterboxRegion(gfx::Rect(frame_size),
  97                                                    content_size);
  98
  99   result.set_x(MakeEven(result.x()));
 100   result.set_y(MakeEven(result.y()));
 101   result.set_width(std::max(kMinFrameWidth, MakeEven(result.width())));
 102   result.set_height(std::max(kMinFrameHeight, MakeEven(result.height())));
 103
 104   return result;
 105 }
 106
 107 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
 108                           const base::Closure& callback) {
 109   render_thread.reset();
 110
 111   // After thread join call the callback on UI thread.
 112   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback);
 113 }
 114
 115 // Responsible for logging the effective frame rate.
 116 class VideoFrameDeliveryLog {
 117  public:
 118   VideoFrameDeliveryLog();
 119
 120   // Report that the frame posted with |frame_time| has been delivered.
 121   void ChronicleFrameDelivery(base::TimeTicks frame_time);
 122
 123  private:
 124   // The following keep track of and log the effective frame rate whenever
 125   // verbose logging is turned on.
 126   base::TimeTicks last_frame_rate_log_time_;
 127   int count_frames_rendered_;
 128
 129   DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
 130 };
 131
 132 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
 133 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
 134 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
 135  public:
 136   FrameSubscriber(VideoCaptureOracle::Event event_type,
 137                   const scoped_refptr<ThreadSafeCaptureOracle>& oracle,
 138                   VideoFrameDeliveryLog* delivery_log)
 139       : event_type_(event_type),
 140         oracle_proxy_(oracle),
 141         delivery_log_(delivery_log) {}
 142
 143   virtual bool ShouldCaptureFrame(
 144       const gfx::Rect& damage_rect,
 145       base::TimeTicks present_time,
 146       scoped_refptr<media::VideoFrame>* storage,
 147       RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
 148           deliver_frame_cb) OVERRIDE;
 149
 150  private:
 151   const VideoCaptureOracle::Event event_type_;
 152   scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
 153   VideoFrameDeliveryLog* const delivery_log_;
 154 };
 155
 156 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
 157 // whose content is updating, a subscriber that is deciding which of these
 158 // updates to capture (and where to deliver them to), and a callback that
 159 // knows how to do the capture and prepare the result for delivery.
 160 //
 161 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
 162 // the RenderWidgetHostView, to process updates that occur via accelerated
 163 // compositing, (b) installing itself as an observer of updates to the
 164 // RenderWidgetHost's backing store, to hook updates that occur via software
 165 // rendering, and (c) running a timer to possibly initiate non-event-driven
 166 // captures that the subscriber might request.
 167 //
 168 // All of this happens on the UI thread, although the
 169 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates
 170 // autonomously on some other thread.
 171 class ContentCaptureSubscription : public content::NotificationObserver {
 172  public:
 173   typedef base::Callback<
 174       void(const base::TimeTicks&,
 175            const scoped_refptr<media::VideoFrame>&,
 176            const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&)>
 177       CaptureCallback;
 178
 179   // Create a subscription. Whenever a manual capture is required, the
 180   // subscription will invoke |capture_callback| on the UI thread to do the
 181   // work.
 182   ContentCaptureSubscription(
 183       const RenderWidgetHost& source,
 184       const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 185       const CaptureCallback& capture_callback);
 186   virtual ~ContentCaptureSubscription();
 187
 188   // content::NotificationObserver implementation.
 189   virtual void Observe(int type,
 190                        const content::NotificationSource& source,
 191                        const content::NotificationDetails& details) OVERRIDE;
 192
 193  private:
 194   void OnTimer();
 195
 196   const int render_process_id_;
 197   const int render_view_id_;
 198
 199   VideoFrameDeliveryLog delivery_log_;
 200   FrameSubscriber paint_subscriber_;
 201   FrameSubscriber timer_subscriber_;
 202   content::NotificationRegistrar registrar_;
 203   CaptureCallback capture_callback_;
 204   base::Timer timer_;
 205
 206   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 207 };
 208
 209 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
 210 // invoke |done_cb| to indicate success or failure. |input| is expected to be
 211 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
 212 // Scaling and letterboxing will be done as needed.
 213 //
 214 // This software implementation should be used only when GPU acceleration of
 215 // these activities is not possible. This operation may be expensive (tens to
 216 // hundreds of milliseconds), so the caller should ensure that it runs on a
 217 // thread where such a pause would cause UI jank.
 218 void RenderVideoFrame(const SkBitmap& input,
 219                       const scoped_refptr<media::VideoFrame>& output,
 220                       const base::Callback<void(bool)>& done_cb);
 221
 222 // Keeps track of the RenderView to be sourced, and executes copying of the
 223 // backing store on the UI BrowserThread.
 224 //
 225 // TODO(nick): It would be nice to merge this with WebContentsTracker, but its
 226 // implementation is currently asynchronous -- in our case, the "rvh changed"
 227 // notification would get posted back to the UI thread and processed later, and
 228 // this seems disadvantageous.
 229 class WebContentsCaptureMachine
 230     : public VideoCaptureMachine,
 231       public WebContentsObserver {
 232  public:
 233   WebContentsCaptureMachine(int render_process_id, int main_render_frame_id);
 234   virtual ~WebContentsCaptureMachine();
 235
 236   // VideoCaptureMachine overrides.
 237   virtual bool Start(const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 238                      const media::VideoCaptureParams& params) OVERRIDE;
 239   virtual void Stop(const base::Closure& callback) OVERRIDE;
 240
 241   // Starts a copy from the backing store or the composited surface. Must be run
 242   // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation
 243   // completes. The copy will occur to |target|.
 244   //
 245   // This may be used as a ContentCaptureSubscription::CaptureCallback.
 246   void Capture(const base::TimeTicks& start_time,
 247                const scoped_refptr<media::VideoFrame>& target,
 248                const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 249                    deliver_frame_cb);
 250
 251   // content::WebContentsObserver implementation.
 252   virtual void DidShowFullscreenWidget(int routing_id) OVERRIDE {
 253     fullscreen_widget_id_ = routing_id;
 254     RenewFrameSubscription();
 255   }
 256
 257   virtual void DidDestroyFullscreenWidget(int routing_id) OVERRIDE {
 258     DCHECK_EQ(fullscreen_widget_id_, routing_id);
 259     fullscreen_widget_id_ = MSG_ROUTING_NONE;
 260     RenewFrameSubscription();
 261   }
 262
 263   virtual void RenderViewReady() OVERRIDE {
 264     RenewFrameSubscription();
 265   }
 266
 267   virtual void AboutToNavigateRenderView(RenderViewHost* rvh) OVERRIDE {
 268     RenewFrameSubscription();
 269   }
 270
 271   virtual void DidNavigateMainFrame(
 272       const LoadCommittedDetails& details,
 273       const FrameNavigateParams& params) OVERRIDE {
 274     RenewFrameSubscription();
 275   }
 276
 277   virtual void WebContentsDestroyed() OVERRIDE;
 278
 279  private:
 280   // Computes the preferred size of the target RenderWidget for optimal capture.
 281   gfx::Size ComputeOptimalTargetSize() const;
 282
 283   // Starts observing the web contents, returning false if lookup fails.
 284   bool StartObservingWebContents();
 285
 286   // Helper function to determine the view that we are currently tracking.
 287   RenderWidgetHost* GetTarget() const;
 288
 289   // Response callback for RenderWidgetHost::CopyFromBackingStore().
 290   void DidCopyFromBackingStore(
 291       const base::TimeTicks& start_time,
 292       const scoped_refptr<media::VideoFrame>& target,
 293       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 294           deliver_frame_cb,
 295       bool success,
 296       const SkBitmap& bitmap);
 297
 298   // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
 299   void DidCopyFromCompositingSurfaceToVideoFrame(
 300       const base::TimeTicks& start_time,
 301       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 302           deliver_frame_cb,
 303       bool success);
 304
 305   // Remove the old subscription, and start a new one. This should be called
 306   // after any change to the WebContents that affects the RenderWidgetHost or
 307   // attached views.
 308   void RenewFrameSubscription();
 309
 310   // Parameters saved in constructor.
 311   const int initial_render_process_id_;
 312   const int initial_main_render_frame_id_;
 313
 314   // A dedicated worker thread on which SkBitmap->VideoFrame conversion will
 315   // occur. Only used when this activity cannot be done on the GPU.
 316   scoped_ptr<base::Thread> render_thread_;
 317
 318   // Makes all the decisions about which frames to copy, and how.
 319   scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
 320
 321   // Video capture parameters that this machine is started with.
 322   media::VideoCaptureParams capture_params_;
 323
 324   // Routing ID of any active fullscreen render widget or MSG_ROUTING_NONE
 325   // otherwise.
 326   int fullscreen_widget_id_;
 327
 328   // Last known RenderView size.
 329   gfx::Size last_view_size_;
 330
 331   // Responsible for forwarding events from the active RenderWidgetHost to the
 332   // oracle, and initiating captures accordingly.
 333   scoped_ptr<ContentCaptureSubscription> subscription_;
 334
 335   // Weak pointer factory used to invalidate callbacks.
 336   // NOTE: Weak pointers must be invalidated before all other member variables.
 337   base::WeakPtrFactory<WebContentsCaptureMachine> weak_ptr_factory_;
 338
 339   DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine);
 340 };
 341
 342 bool FrameSubscriber::ShouldCaptureFrame(
 343     const gfx::Rect& damage_rect,
 344     base::TimeTicks present_time,
 345     scoped_refptr<media::VideoFrame>* storage,
 346     DeliverFrameCallback* deliver_frame_cb) {
 347   TRACE_EVENT1("mirroring", "FrameSubscriber::ShouldCaptureFrame",
 348                "instance", this);
 349
 350   ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
 351   bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture(
 352       event_type_, damage_rect, present_time, storage, &capture_frame_cb);
 353
 354   if (!capture_frame_cb.is_null())
 355     *deliver_frame_cb = base::Bind(capture_frame_cb, *storage);
 356   if (oracle_decision)
 357     delivery_log_->ChronicleFrameDelivery(present_time);
 358   return oracle_decision;
 359 }
 360
 361 ContentCaptureSubscription::ContentCaptureSubscription(
 362     const RenderWidgetHost& source,
 363     const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 364     const CaptureCallback& capture_callback)
 365     : render_process_id_(source.GetProcess()->GetID()),
 366       render_view_id_(source.GetRoutingID()),
 367       delivery_log_(),
 368       paint_subscriber_(VideoCaptureOracle::kSoftwarePaint, oracle_proxy,
 369                         &delivery_log_),
 370       timer_subscriber_(VideoCaptureOracle::kTimerPoll, oracle_proxy,
 371                         &delivery_log_),
 372       capture_callback_(capture_callback),
 373       timer_(true, true) {
 374   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 375
 376   RenderWidgetHostViewBase* view = static_cast<RenderWidgetHostViewBase*>(
 377       source.GetView());
 378
 379   // Subscribe to accelerated presents. These will be serviced directly by the
 380   // oracle.
 381   if (view && kAcceleratedSubscriberIsSupported) {
 382     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
 383         new FrameSubscriber(VideoCaptureOracle::kCompositorUpdate,
 384             oracle_proxy, &delivery_log_));
 385     view->BeginFrameSubscription(subscriber.Pass());
 386   }
 387
 388   // Subscribe to software paint events. This instance will service these by
 389   // reflecting them back to the WebContentsCaptureMachine via
 390   // |capture_callback|.
 391   registrar_.Add(
 392       this, content::NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE,
 393       Source<RenderWidgetHost>(&source));
 394
 395   // Subscribe to timer events. This instance will service these as well.
 396   timer_.Start(FROM_HERE, oracle_proxy->min_capture_period(),
 397                base::Bind(&ContentCaptureSubscription::OnTimer,
 398                           base::Unretained(this)));
 399 }
 400
 401 ContentCaptureSubscription::~ContentCaptureSubscription() {
 402   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 403   if (kAcceleratedSubscriberIsSupported) {
 404     RenderViewHost* source = RenderViewHost::FromID(render_process_id_,
 405                                                     render_view_id_);
 406     if (source) {
 407       RenderWidgetHostViewBase* view = static_cast<RenderWidgetHostViewBase*>(
 408           source->GetView());
 409       if (view)
 410         view->EndFrameSubscription();
 411     }
 412   }
 413 }
 414
 415 void ContentCaptureSubscription::Observe(
 416     int type,
 417     const content::NotificationSource& source,
 418     const content::NotificationDetails& details) {
 419   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 420   DCHECK_EQ(NOTIFICATION_RENDER_WIDGET_HOST_DID_UPDATE_BACKING_STORE, type);
 421
 422   RenderWidgetHostImpl* rwh =
 423       RenderWidgetHostImpl::From(Source<RenderWidgetHost>(source).ptr());
 424
 425   // This message occurs on window resizes and visibility changes even when
 426   // accelerated compositing is active, so we need to filter out these cases.
 427   if (!rwh || !rwh->GetView())
 428     return;
 429   // Mac sends DID_UPDATE_BACKING_STORE messages to inform the capture system
 430   // of new software compositor frames, so always treat these messages as
 431   // signals of a new frame on Mac.
 432   // http://crbug.com/333986
 433 #if !defined(OS_MACOSX)
 434   if (rwh->GetView()->IsSurfaceAvailableForCopy())
 435     return;
 436 #endif
 437
 438   TRACE_EVENT1("mirroring", "ContentCaptureSubscription::Observe",
 439                "instance", this);
 440
 441   base::Closure copy_done_callback;
 442   scoped_refptr<media::VideoFrame> frame;
 443   RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
 444   const base::TimeTicks start_time = base::TimeTicks::Now();
 445   if (paint_subscriber_.ShouldCaptureFrame(gfx::Rect(),
 446                                            start_time,
 447                                            &frame,
 448                                            &deliver_frame_cb)) {
 449     // This message happens just before paint. If we post a task to do the copy,
 450     // it should run soon after the paint.
 451     BrowserThread::PostTask(
 452         BrowserThread::UI, FROM_HERE,
 453         base::Bind(capture_callback_, start_time, frame, deliver_frame_cb));
 454   }
 455 }
 456
 457 void ContentCaptureSubscription::OnTimer() {
 458   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 459   TRACE_EVENT0("mirroring", "ContentCaptureSubscription::OnTimer");
 460
 461   scoped_refptr<media::VideoFrame> frame;
 462   RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
 463
 464   const base::TimeTicks start_time = base::TimeTicks::Now();
 465   if (timer_subscriber_.ShouldCaptureFrame(gfx::Rect(),
 466                                            start_time,
 467                                            &frame,
 468                                            &deliver_frame_cb)) {
 469     capture_callback_.Run(start_time, frame, deliver_frame_cb);
 470   }
 471 }
 472
 473 void RenderVideoFrame(const SkBitmap& input,
 474                       const scoped_refptr<media::VideoFrame>& output,
 475                       const base::Callback<void(bool)>& done_cb) {
 476   base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
 477
 478   SkAutoLockPixels locker(input);
 479
 480   // Sanity-check the captured bitmap.
 481   if (input.empty() ||
 482       !input.readyToDraw() ||
 483       input.colorType() != kN32_SkColorType ||
 484       input.width() < 2 || input.height() < 2) {
 485     DVLOG(1) << "input unacceptable (size="
 486              << input.getSize()
 487              << ", ready=" << input.readyToDraw()
 488              << ", colorType=" << input.colorType() << ')';
 489     return;
 490   }
 491
 492   // Sanity-check the output buffer.
 493   if (output->format() != media::VideoFrame::I420) {
 494     NOTREACHED();
 495     return;
 496   }
 497
 498   // Calculate the width and height of the content region in the |output|, based
 499   // on the aspect ratio of |input|.
 500   gfx::Rect region_in_frame = ComputeYV12LetterboxRegion(
 501       output->coded_size(), gfx::Size(input.width(), input.height()));
 502
 503   // Scale the bitmap to the required size, if necessary.
 504   SkBitmap scaled_bitmap;
 505   if (input.width() != region_in_frame.width() ||
 506       input.height() != region_in_frame.height()) {
 507
 508     skia::ImageOperations::ResizeMethod method;
 509     if (input.width() < region_in_frame.width() ||
 510         input.height() < region_in_frame.height()) {
 511       // Avoid box filtering when magnifying, because it's actually
 512       // nearest-neighbor.
 513       method = skia::ImageOperations::RESIZE_HAMMING1;
 514     } else {
 515       method = skia::ImageOperations::RESIZE_BOX;
 516     }
 517
 518     TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "Scale");
 519     scaled_bitmap = skia::ImageOperations::Resize(input, method,
 520                                                   region_in_frame.width(),
 521                                                   region_in_frame.height());
 522   } else {
 523     scaled_bitmap = input;
 524   }
 525
 526   TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", output.get(), "YUV");
 527   {
 528     SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap);
 529
 530     media::CopyRGBToVideoFrame(
 531         reinterpret_cast<uint8*>(scaled_bitmap.getPixels()),
 532         scaled_bitmap.rowBytes(),
 533         region_in_frame,
 534         output.get());
 535   }
 536
 537   // The result is now ready.
 538   ignore_result(failure_handler.Release());
 539   done_cb.Run(true);
 540 }
 541
 542 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
 543     : last_frame_rate_log_time_(),
 544       count_frames_rendered_(0) {
 545 }
 546
 547 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time) {
 548   // Log frame rate, if verbose logging is turned on.
 549   static const base::TimeDelta kFrameRateLogInterval =
 550       base::TimeDelta::FromSeconds(10);
 551   if (last_frame_rate_log_time_.is_null()) {
 552     last_frame_rate_log_time_ = frame_time;
 553     count_frames_rendered_ = 0;
 554   } else {
 555     ++count_frames_rendered_;
 556     const base::TimeDelta elapsed = frame_time - last_frame_rate_log_time_;
 557     if (elapsed >= kFrameRateLogInterval) {
 558       const double measured_fps =
 559           count_frames_rendered_ / elapsed.InSecondsF();
 560       UMA_HISTOGRAM_COUNTS(
 561           "TabCapture.FrameRate",
 562           static_cast<int>(measured_fps));
 563       VLOG(1) << "Current measured frame rate for "
 564               << "WebContentsVideoCaptureDevice is " << measured_fps << " FPS.";
 565       last_frame_rate_log_time_ = frame_time;
 566       count_frames_rendered_ = 0;
 567     }
 568   }
 569 }
 570
 571 WebContentsCaptureMachine::WebContentsCaptureMachine(int render_process_id,
 572                                                      int main_render_frame_id)
 573     : initial_render_process_id_(render_process_id),
 574       initial_main_render_frame_id_(main_render_frame_id),
 575       fullscreen_widget_id_(MSG_ROUTING_NONE),
 576       weak_ptr_factory_(this) {}
 577
 578 WebContentsCaptureMachine::~WebContentsCaptureMachine() {
 579   BrowserThread::PostBlockingPoolTask(
 580       FROM_HERE,
 581       base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_),
 582                  base::Bind(&base::DoNothing)));
 583 }
 584
 585 bool WebContentsCaptureMachine::Start(
 586     const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 587     const media::VideoCaptureParams& params) {
 588   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 589   DCHECK(!started_);
 590
 591   DCHECK(oracle_proxy.get());
 592   oracle_proxy_ = oracle_proxy;
 593   capture_params_ = params;
 594
 595   render_thread_.reset(new base::Thread("WebContentsVideo_RenderThread"));
 596   if (!render_thread_->Start()) {
 597     DVLOG(1) << "Failed to spawn render thread.";
 598     render_thread_.reset();
 599     return false;
 600   }
 601
 602   if (!StartObservingWebContents()) {
 603     DVLOG(1) << "Failed to observe web contents.";
 604     render_thread_.reset();
 605     return false;
 606   }
 607
 608   started_ = true;
 609   return true;
 610 }
 611
 612 void WebContentsCaptureMachine::Stop(const base::Closure& callback) {
 613   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 614   subscription_.reset();
 615   if (web_contents()) {
 616     web_contents()->DecrementCapturerCount();
 617     Observe(NULL);
 618   }
 619
 620   // Any callback that intend to use render_thread_ will not work after it is
 621   // passed.
 622   weak_ptr_factory_.InvalidateWeakPtrs();
 623
 624   // The render thread cannot be stopped on the UI thread, so post a message
 625   // to the thread pool used for blocking operations.
 626   BrowserThread::PostBlockingPoolTask(
 627       FROM_HERE,
 628       base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_),
 629                  callback));
 630
 631   started_ = false;
 632 }
 633
 634 void WebContentsCaptureMachine::Capture(
 635     const base::TimeTicks& start_time,
 636     const scoped_refptr<media::VideoFrame>& target,
 637     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 638         deliver_frame_cb) {
 639   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 640
 641   RenderWidgetHost* rwh = GetTarget();
 642   RenderWidgetHostViewBase* view =
 643       rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL;
 644   if (!view || !rwh) {
 645     deliver_frame_cb.Run(base::TimeTicks(), false);
 646     return;
 647   }
 648
 649   gfx::Size video_size = target->coded_size();
 650   gfx::Size view_size = view->GetViewBounds().size();
 651   gfx::Size fitted_size;
 652   if (!view_size.IsEmpty()) {
 653     fitted_size = ComputeYV12LetterboxRegion(video_size, view_size).size();
 654   }
 655   if (view_size != last_view_size_) {
 656     last_view_size_ = view_size;
 657
 658     // Measure the number of kilopixels.
 659     UMA_HISTOGRAM_COUNTS_10000(
 660         "TabCapture.ViewChangeKiloPixels",
 661         view_size.width() * view_size.height() / 1024);
 662   }
 663
 664   if (view->CanCopyToVideoFrame()) {
 665     view->CopyFromCompositingSurfaceToVideoFrame(
 666         gfx::Rect(view_size),
 667         target,
 668         base::Bind(&WebContentsCaptureMachine::
 669                         DidCopyFromCompositingSurfaceToVideoFrame,
 670                    weak_ptr_factory_.GetWeakPtr(),
 671                    start_time, deliver_frame_cb));
 672   } else {
 673     rwh->CopyFromBackingStore(
 674         gfx::Rect(),
 675         fitted_size,  // Size here is a request not always honored.
 676         base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore,
 677                    weak_ptr_factory_.GetWeakPtr(),
 678                    start_time,
 679                    target,
 680                    deliver_frame_cb),
 681         kN32_SkColorType);
 682   }
 683 }
 684
 685 gfx::Size WebContentsCaptureMachine::ComputeOptimalTargetSize() const {
 686   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 687
 688   gfx::Size optimal_size = oracle_proxy_->GetCaptureSize();
 689
 690   // If the ratio between physical and logical pixels is greater than 1:1,
 691   // shrink |optimal_size| by that amount.  Then, when external code resizes the
 692   // render widget to the "preferred size," the widget will be physically
 693   // rendered at the exact capture size, thereby eliminating unnecessary scaling
 694   // operations in the graphics pipeline.
 695   RenderWidgetHost* const rwh = GetTarget();
 696   RenderWidgetHostView* const rwhv = rwh ? rwh->GetView() : NULL;
 697   if (rwhv) {
 698     const gfx::NativeView view = rwhv->GetNativeView();
 699     gfx::Screen* const screen = gfx::Screen::GetScreenFor(view);
 700     if (screen->IsDIPEnabled()) {
 701       const gfx::Display display = screen->GetDisplayNearestWindow(view);
 702       const float scale = display.device_scale_factor();
 703       if (scale > 1.0f) {
 704         const gfx::Size shrunk_size(
 705             gfx::ToFlooredSize(gfx::ScaleSize(optimal_size, 1.0f / scale)));
 706         if (shrunk_size.width() > 0 && shrunk_size.height() > 0)
 707           optimal_size = shrunk_size;
 708       }
 709     }
 710   }
 711
 712   VLOG(1) << "Computed optimal target size: " << optimal_size.ToString();
 713   return optimal_size;
 714 }
 715
 716 bool WebContentsCaptureMachine::StartObservingWebContents() {
 717   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 718
 719   // Look-up the RenderFrameHost and, from that, the WebContents that wraps it.
 720   // If successful, begin observing the WebContents instance.
 721   //
 722   // Why this can be unsuccessful: The request for mirroring originates in a
 723   // render process, and this request is based on the current main RenderFrame
 724   // associated with a tab.  However, by the time we get up-and-running here,
 725   // there have been multiple back-and-forth IPCs between processes, as well as
 726   // a bit of indirection across threads.  It's easily possible that, in the
 727   // meantime, the original RenderFrame may have gone away.
 728   Observe(WebContents::FromRenderFrameHost(RenderFrameHost::FromID(
 729       initial_render_process_id_, initial_main_render_frame_id_)));
 730   DVLOG_IF(1, !web_contents())
 731       << "Could not find WebContents associated with main RenderFrameHost "
 732       << "referenced by render_process_id=" << initial_render_process_id_
 733       << ", routing_id=" << initial_main_render_frame_id_;
 734
 735   WebContentsImpl* contents = static_cast<WebContentsImpl*>(web_contents());
 736   if (contents) {
 737     contents->IncrementCapturerCount(ComputeOptimalTargetSize());
 738     fullscreen_widget_id_ = contents->GetFullscreenWidgetRoutingID();
 739     RenewFrameSubscription();
 740     return true;
 741   }
 742   return false;
 743 }
 744
 745 void WebContentsCaptureMachine::WebContentsDestroyed() {
 746   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 747
 748   subscription_.reset();
 749   web_contents()->DecrementCapturerCount();
 750   oracle_proxy_->ReportError("WebContentsDestroyed()");
 751 }
 752
 753 RenderWidgetHost* WebContentsCaptureMachine::GetTarget() const {
 754   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 755   if (!web_contents())
 756     return NULL;
 757
 758   RenderWidgetHost* rwh = NULL;
 759   if (fullscreen_widget_id_ != MSG_ROUTING_NONE) {
 760     RenderProcessHost* process = web_contents()->GetRenderProcessHost();
 761     if (process)
 762       rwh = RenderWidgetHost::FromID(process->GetID(), fullscreen_widget_id_);
 763   } else {
 764     rwh = web_contents()->GetRenderViewHost();
 765   }
 766
 767   return rwh;
 768 }
 769
 770 void WebContentsCaptureMachine::DidCopyFromBackingStore(
 771     const base::TimeTicks& start_time,
 772     const scoped_refptr<media::VideoFrame>& target,
 773     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 774         deliver_frame_cb,
 775     bool success,
 776     const SkBitmap& bitmap) {
 777   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 778
 779   base::TimeTicks now = base::TimeTicks::Now();
 780   DCHECK(render_thread_.get());
 781   if (success) {
 782     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time);
 783     TRACE_EVENT_ASYNC_STEP_INTO0("mirroring", "Capture", target.get(),
 784                                  "Render");
 785     render_thread_->message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
 786         &RenderVideoFrame, bitmap, target,
 787         base::Bind(deliver_frame_cb, start_time)));
 788   } else {
 789     // Capture can fail due to transient issues, so just skip this frame.
 790     DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
 791     deliver_frame_cb.Run(start_time, false);
 792   }
 793 }
 794
 795 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
 796     const base::TimeTicks& start_time,
 797     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
 798         deliver_frame_cb,
 799     bool success) {
 800   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 801   base::TimeTicks now = base::TimeTicks::Now();
 802
 803   if (success) {
 804     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time);
 805   } else {
 806     // Capture can fail due to transient issues, so just skip this frame.
 807     DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
 808   }
 809   deliver_frame_cb.Run(start_time, success);
 810 }
 811
 812 void WebContentsCaptureMachine::RenewFrameSubscription() {
 813   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 814
 815   // Always destroy the old subscription before creating a new one.
 816   subscription_.reset();
 817
 818   RenderWidgetHost* rwh = GetTarget();
 819   if (!rwh || !rwh->GetView())
 820     return;
 821
 822   subscription_.reset(new ContentCaptureSubscription(*rwh, oracle_proxy_,
 823       base::Bind(&WebContentsCaptureMachine::Capture,
 824                  weak_ptr_factory_.GetWeakPtr())));
 825 }
 826
 827 }  // namespace
 828
 829 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice(
 830     int render_process_id, int main_render_frame_id)
 831     : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
 832         new WebContentsCaptureMachine(
 833             render_process_id, main_render_frame_id)))) {}
 834
 835 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() {
 836   DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying.";
 837 }
 838
 839 // static
 840 media::VideoCaptureDevice* WebContentsVideoCaptureDevice::Create(
 841     const std::string& device_id) {
 842   // Parse device_id into render_process_id and render_view_id.
 843   int render_process_id = -1;
 844   int main_render_frame_id = -1;
 845   if (!WebContentsCaptureUtil::ExtractTabCaptureTarget(
 846            device_id, &render_process_id, &main_render_frame_id)) {
 847     return NULL;
 848   }
 849
 850   return new WebContentsVideoCaptureDevice(
 851       render_process_id, main_render_frame_id);
 852 }
 853
 854 void WebContentsVideoCaptureDevice::AllocateAndStart(
 855     const media::VideoCaptureParams& params,
 856     scoped_ptr<Client> client) {
 857   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
 858   core_->AllocateAndStart(params, client.Pass());
 859 }
 860
 861 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
 862   core_->StopAndDeAllocate();
 863 }
 864
 865 }  // namespace content