1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // Implementation notes: This needs to work on a variety of hardware
6 // configurations where the speed of the CPU and GPU greatly affect overall
7 // performance. Spanning several threads, the process of capturing has been
8 // split up into four conceptual stages:
10 // 1. Reserve Buffer: Before a frame can be captured, a slot in the client's
11 // shared-memory IPC buffer is reserved. There are only a few of these;
12 // when they run out, it indicates that the downstream client -- likely a
13 // video encoder -- is the performance bottleneck, and that the rate of
14 // frame capture should be throttled back.
16 // 2. Capture: A bitmap is snapshotted/copied from the RenderWidget's backing
17 // store. This is initiated on the UI BrowserThread, and often occurs
18 // asynchronously. Where supported, the GPU scales and color converts
19 // frames to our desired size, and the readback happens directly into the
20 // shared-memory buffer. But this is not always possible, particularly when
21 // accelerated compositing is disabled.
23 // 3. Render (if needed): If the web contents cannot be captured directly into
24 // our target size and color format, scaling and colorspace conversion must
25 // be done on the CPU. A dedicated thread is used for this operation, to
26 // avoid blocking the UI thread. The Render stage always reads from a
27 // bitmap returned by Capture, and writes into the reserved slot in the
28 // shared-memory buffer.
30 // 4. Deliver: The rendered video frame is returned to the client (which
31 // implements the VideoCaptureDevice::Client interface). Because all
32 // paths have written the frame into the IPC buffer, this step should
33 // never need to do an additional copy of the pixel data.
35 // In the best-performing case, the Render step is bypassed: Capture produces
36 // ready-to-Deliver frames. But when accelerated readback is not possible, the
37 // system is designed so that Capture and Render may run concurrently. A timing
38 // diagram helps illustrate this point (@30 FPS):
40 // Time: 0ms 33ms 66ms 99ms
41 // thread1: |-Capture-f1------v |-Capture-f2------v |-Capture-f3----v |-Capt
42 // thread2: |-Render-f1-----v |-Render-f2-----v |-Render-f3
44 // In the above example, both capturing and rendering *each* take almost the
45 // full 33 ms available between frames, yet we see that the required throughput
48 // Turning on verbose logging will cause the effective frame rate to be logged
49 // at 5-second intervals.
51 #include "content/browser/media/capture/web_contents_video_capture_device.h"
53 #include "base/basictypes.h"
54 #include "base/bind.h"
55 #include "base/callback_helpers.h"
56 #include "base/location.h"
57 #include "base/logging.h"
58 #include "base/memory/scoped_ptr.h"
59 #include "base/memory/weak_ptr.h"
60 #include "base/metrics/histogram.h"
61 #include "base/sequenced_task_runner.h"
62 #include "base/single_thread_task_runner.h"
63 #include "base/threading/thread.h"
64 #include "base/threading/thread_checker.h"
65 #include "base/time/time.h"
66 #include "content/browser/media/capture/web_contents_capture_util.h"
67 #include "content/browser/media/capture/web_contents_tracker.h"
68 #include "content/browser/renderer_host/render_widget_host_impl.h"
69 #include "content/browser/renderer_host/render_widget_host_view_base.h"
70 #include "content/public/browser/browser_thread.h"
71 #include "content/public/browser/render_process_host.h"
72 #include "content/public/browser/render_widget_host_view.h"
73 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
74 #include "content/public/browser/web_contents.h"
75 #include "media/base/video_capture_types.h"
76 #include "media/base/video_util.h"
77 #include "media/capture/content/screen_capture_device_core.h"
78 #include "media/capture/content/thread_safe_capture_oracle.h"
79 #include "media/capture/content/video_capture_oracle.h"
80 #include "skia/ext/image_operations.h"
81 #include "third_party/skia/include/core/SkBitmap.h"
82 #include "third_party/skia/include/core/SkColor.h"
83 #include "ui/base/layout.h"
84 #include "ui/gfx/geometry/dip_util.h"
85 #include "ui/gfx/geometry/size_conversions.h"
91 void DeleteOnWorkerThread(scoped_ptr
<base::Thread
> render_thread
,
92 const base::Closure
& callback
) {
93 render_thread
.reset();
95 // After thread join call the callback on UI thread.
96 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, callback
);
99 // Responsible for logging the effective frame rate.
100 class VideoFrameDeliveryLog
{
102 VideoFrameDeliveryLog();
104 // Report that the frame posted with |frame_time| has been delivered.
105 void ChronicleFrameDelivery(base::TimeTicks frame_time
);
108 // The following keep track of and log the effective frame rate whenever
109 // verbose logging is turned on.
110 base::TimeTicks last_frame_rate_log_time_
;
111 int count_frames_rendered_
;
113 DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog
);
116 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
117 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
118 class FrameSubscriber
: public RenderWidgetHostViewFrameSubscriber
{
120 FrameSubscriber(media::VideoCaptureOracle::Event event_type
,
121 const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle
,
122 VideoFrameDeliveryLog
* delivery_log
)
123 : event_type_(event_type
),
124 oracle_proxy_(oracle
),
125 delivery_log_(delivery_log
) {}
127 bool ShouldCaptureFrame(
128 const gfx::Rect
& damage_rect
,
129 base::TimeTicks present_time
,
130 scoped_refptr
<media::VideoFrame
>* storage
,
131 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
*
132 deliver_frame_cb
) override
;
135 const media::VideoCaptureOracle::Event event_type_
;
136 scoped_refptr
<media::ThreadSafeCaptureOracle
> oracle_proxy_
;
137 VideoFrameDeliveryLog
* const delivery_log_
;
140 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
141 // whose content is updating, a subscriber that is deciding which of these
142 // updates to capture (and where to deliver them to), and a callback that
143 // knows how to do the capture and prepare the result for delivery.
145 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
146 // the RenderWidgetHostView, to process compositor updates, and (b) running a
147 // timer to possibly initiate forced, non-event-driven captures needed by
148 // downstream consumers that require frame repeats of unchanged content.
150 // All of this happens on the UI thread, although the
151 // RenderWidgetHostViewFrameSubscriber we install may be dispatching updates
152 // autonomously on some other thread.
153 class ContentCaptureSubscription
{
155 typedef base::Callback
<
156 void(const base::TimeTicks
&,
157 const scoped_refptr
<media::VideoFrame
>&,
158 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&)>
161 // Create a subscription. Whenever a manual capture is required, the
162 // subscription will invoke |capture_callback| on the UI thread to do the
164 ContentCaptureSubscription(
165 const RenderWidgetHost
& source
,
166 const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle_proxy
,
167 const CaptureCallback
& capture_callback
);
168 ~ContentCaptureSubscription();
173 // Maintain a weak reference to the RenderWidgetHost (via its routing ID),
174 // since the instance could be destroyed externally during the lifetime of
176 const int render_process_id_
;
177 const int render_widget_id_
;
179 VideoFrameDeliveryLog delivery_log_
;
180 FrameSubscriber timer_subscriber_
;
181 CaptureCallback capture_callback_
;
184 DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription
);
187 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
188 // invoke |done_cb| to indicate success or failure. |input| is expected to be
189 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
190 // Scaling and letterboxing will be done as needed.
192 // This software implementation should be used only when GPU acceleration of
193 // these activities is not possible. This operation may be expensive (tens to
194 // hundreds of milliseconds), so the caller should ensure that it runs on a
195 // thread where such a pause would cause UI jank.
196 void RenderVideoFrame(const SkBitmap
& input
,
197 const scoped_refptr
<media::VideoFrame
>& output
,
198 const base::Callback
<void(bool)>& done_cb
);
200 // Renews capture subscriptions based on feedback from WebContentsTracker, and
201 // also executes copying of the backing store on the UI BrowserThread.
202 class WebContentsCaptureMachine
: public media::VideoCaptureMachine
{
204 WebContentsCaptureMachine(int render_process_id
,
205 int main_render_frame_id
,
206 bool enable_auto_throttling
);
207 ~WebContentsCaptureMachine() override
;
209 // VideoCaptureMachine overrides.
210 void Start(const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle_proxy
,
211 const media::VideoCaptureParams
& params
,
212 const base::Callback
<void(bool)> callback
) override
;
213 void Stop(const base::Closure
& callback
) override
;
214 bool IsAutoThrottlingEnabled() const override
{
215 return auto_throttling_enabled_
;
218 // Starts a copy from the backing store or the composited surface. Must be run
219 // on the UI BrowserThread. |deliver_frame_cb| will be run when the operation
220 // completes. The copy will occur to |target|.
222 // This may be used as a ContentCaptureSubscription::CaptureCallback.
223 void Capture(const base::TimeTicks
& start_time
,
224 const scoped_refptr
<media::VideoFrame
>& target
,
225 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&
230 const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle_proxy
,
231 const media::VideoCaptureParams
& params
);
232 void InternalStop(const base::Closure
& callback
);
233 bool IsStarted() const;
235 // Computes the preferred size of the target RenderWidget for optimal capture.
236 gfx::Size
ComputeOptimalViewSize() const;
238 // Response callback for RenderWidgetHost::CopyFromBackingStore().
239 void DidCopyFromBackingStore(
240 const base::TimeTicks
& start_time
,
241 const scoped_refptr
<media::VideoFrame
>& target
,
242 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&
244 const SkBitmap
& bitmap
,
245 ReadbackResponse response
);
247 // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
248 void DidCopyFromCompositingSurfaceToVideoFrame(
249 const base::TimeTicks
& start_time
,
250 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&
254 // Remove the old subscription, and attempt to start a new one if |had_target|
256 void RenewFrameSubscription(bool had_target
);
258 // Called whenever the render widget is resized.
259 void UpdateCaptureSize();
261 // Parameters saved in constructor.
262 const int initial_render_process_id_
;
263 const int initial_main_render_frame_id_
;
265 // Tracks events and calls back to RenewFrameSubscription() to maintain
266 // capture on the correct RenderWidgetHost.
267 const scoped_refptr
<WebContentsTracker
> tracker_
;
269 // Set to false to prevent the capture size from automatically adjusting in
270 // response to end-to-end utilization. This is enabled via the throttling
271 // option in the WebContentsVideoCaptureDevice device ID.
272 const bool auto_throttling_enabled_
;
274 // A dedicated worker thread on which SkBitmap->VideoFrame conversion will
275 // occur. Only used when this activity cannot be done on the GPU.
276 scoped_ptr
<base::Thread
> render_thread_
;
278 // Makes all the decisions about which frames to copy, and how.
279 scoped_refptr
<media::ThreadSafeCaptureOracle
> oracle_proxy_
;
281 // Video capture parameters that this machine is started with.
282 media::VideoCaptureParams capture_params_
;
284 // Last known RenderView size.
285 gfx::Size last_view_size_
;
287 // Responsible for forwarding events from the active RenderWidgetHost to the
288 // oracle, and initiating captures accordingly.
289 scoped_ptr
<ContentCaptureSubscription
> subscription_
;
291 // Weak pointer factory used to invalidate callbacks.
292 // NOTE: Weak pointers must be invalidated before all other member variables.
293 base::WeakPtrFactory
<WebContentsCaptureMachine
> weak_ptr_factory_
;
295 DISALLOW_COPY_AND_ASSIGN(WebContentsCaptureMachine
);
298 bool FrameSubscriber::ShouldCaptureFrame(
299 const gfx::Rect
& damage_rect
,
300 base::TimeTicks present_time
,
301 scoped_refptr
<media::VideoFrame
>* storage
,
302 DeliverFrameCallback
* deliver_frame_cb
) {
303 TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame",
306 media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb
;
307 bool oracle_decision
= oracle_proxy_
->ObserveEventAndDecideCapture(
308 event_type_
, damage_rect
, present_time
, storage
, &capture_frame_cb
);
310 if (!capture_frame_cb
.is_null())
311 *deliver_frame_cb
= base::Bind(capture_frame_cb
, *storage
);
313 delivery_log_
->ChronicleFrameDelivery(present_time
);
314 return oracle_decision
;
317 ContentCaptureSubscription::ContentCaptureSubscription(
318 const RenderWidgetHost
& source
,
319 const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle_proxy
,
320 const CaptureCallback
& capture_callback
)
321 : render_process_id_(source
.GetProcess()->GetID()),
322 render_widget_id_(source
.GetRoutingID()),
324 timer_subscriber_(media::VideoCaptureOracle::kTimerPoll
, oracle_proxy
,
326 capture_callback_(capture_callback
),
328 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
330 RenderWidgetHostView
* const view
= source
.GetView();
332 // Subscribe to compositor updates. These will be serviced directly by the
335 scoped_ptr
<RenderWidgetHostViewFrameSubscriber
> subscriber(
336 new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate
,
337 oracle_proxy
, &delivery_log_
));
338 view
->BeginFrameSubscription(subscriber
.Pass());
341 // Subscribe to timer events. This instance will service these as well.
342 timer_
.Start(FROM_HERE
,
343 std::max(oracle_proxy
->min_capture_period(),
344 base::TimeDelta::FromMilliseconds(media
345 ::VideoCaptureOracle::kMinTimerPollPeriodMillis
)),
346 base::Bind(&ContentCaptureSubscription::OnTimer
,
347 base::Unretained(this)));
350 ContentCaptureSubscription::~ContentCaptureSubscription() {
351 // If the BrowserThreads have been torn down, then the browser is in the final
352 // stages of exiting and it is dangerous to take any further action. We must
353 // return early. http://crbug.com/396413
354 if (!BrowserThread::IsMessageLoopValid(BrowserThread::UI
))
357 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
358 RenderWidgetHost
* const source
=
359 RenderWidgetHost::FromID(render_process_id_
, render_widget_id_
);
360 RenderWidgetHostView
* const view
= source
? source
->GetView() : NULL
;
362 view
->EndFrameSubscription();
365 void ContentCaptureSubscription::OnTimer() {
366 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
367 TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnTimer");
369 scoped_refptr
<media::VideoFrame
> frame
;
370 RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb
;
372 const base::TimeTicks start_time
= base::TimeTicks::Now();
373 if (timer_subscriber_
.ShouldCaptureFrame(gfx::Rect(),
376 &deliver_frame_cb
)) {
377 capture_callback_
.Run(start_time
, frame
, deliver_frame_cb
);
381 void RenderVideoFrame(const SkBitmap
& input
,
382 const scoped_refptr
<media::VideoFrame
>& output
,
383 const base::Callback
<void(bool)>& done_cb
) {
384 base::ScopedClosureRunner
failure_handler(base::Bind(done_cb
, false));
386 SkAutoLockPixels
locker(input
);
388 // Sanity-check the captured bitmap.
390 !input
.readyToDraw() ||
391 input
.colorType() != kN32_SkColorType
||
392 input
.width() < 2 || input
.height() < 2) {
393 DVLOG(1) << "input unacceptable (size="
395 << ", ready=" << input
.readyToDraw()
396 << ", colorType=" << input
.colorType() << ')';
400 // Sanity-check the output buffer.
401 if (output
->format() != media::PIXEL_FORMAT_I420
) {
406 // Calculate the width and height of the content region in the |output|, based
407 // on the aspect ratio of |input|.
408 const gfx::Rect region_in_frame
= media::ComputeLetterboxRegion(
409 output
->visible_rect(), gfx::Size(input
.width(), input
.height()));
411 // Scale the bitmap to the required size, if necessary.
412 SkBitmap scaled_bitmap
;
413 if (input
.width() != region_in_frame
.width() ||
414 input
.height() != region_in_frame
.height()) {
416 skia::ImageOperations::ResizeMethod method
;
417 if (input
.width() < region_in_frame
.width() ||
418 input
.height() < region_in_frame
.height()) {
419 // Avoid box filtering when magnifying, because it's actually
421 method
= skia::ImageOperations::RESIZE_HAMMING1
;
423 method
= skia::ImageOperations::RESIZE_BOX
;
426 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture",
427 "Capture", output
.get(), "Scale");
428 scaled_bitmap
= skia::ImageOperations::Resize(input
, method
,
429 region_in_frame
.width(),
430 region_in_frame
.height());
432 scaled_bitmap
= input
;
435 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", output
.get(), "YUV");
437 // Align to 2x2 pixel boundaries, as required by
438 // media::CopyRGBToVideoFrame().
439 const gfx::Rect
region_in_yv12_frame(region_in_frame
.x() & ~1,
440 region_in_frame
.y() & ~1,
441 region_in_frame
.width() & ~1,
442 region_in_frame
.height() & ~1);
443 if (region_in_yv12_frame
.IsEmpty())
446 SkAutoLockPixels
scaled_bitmap_locker(scaled_bitmap
);
447 media::CopyRGBToVideoFrame(
448 reinterpret_cast<uint8
*>(scaled_bitmap
.getPixels()),
449 scaled_bitmap
.rowBytes(),
450 region_in_yv12_frame
,
454 // The result is now ready.
455 ignore_result(failure_handler
.Release());
459 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
460 : last_frame_rate_log_time_(),
461 count_frames_rendered_(0) {
464 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time
) {
465 // Log frame rate, if verbose logging is turned on.
466 static const base::TimeDelta kFrameRateLogInterval
=
467 base::TimeDelta::FromSeconds(10);
468 if (last_frame_rate_log_time_
.is_null()) {
469 last_frame_rate_log_time_
= frame_time
;
470 count_frames_rendered_
= 0;
472 ++count_frames_rendered_
;
473 const base::TimeDelta elapsed
= frame_time
- last_frame_rate_log_time_
;
474 if (elapsed
>= kFrameRateLogInterval
) {
475 const double measured_fps
=
476 count_frames_rendered_
/ elapsed
.InSecondsF();
477 UMA_HISTOGRAM_COUNTS(
478 "TabCapture.FrameRate",
479 static_cast<int>(measured_fps
));
480 VLOG(1) << "Current measured frame rate for "
481 << "WebContentsVideoCaptureDevice is " << measured_fps
<< " FPS.";
482 last_frame_rate_log_time_
= frame_time
;
483 count_frames_rendered_
= 0;
488 WebContentsCaptureMachine::WebContentsCaptureMachine(
489 int render_process_id
,
490 int main_render_frame_id
,
491 bool enable_auto_throttling
)
492 : initial_render_process_id_(render_process_id
),
493 initial_main_render_frame_id_(main_render_frame_id
),
494 tracker_(new WebContentsTracker(true)),
495 auto_throttling_enabled_(enable_auto_throttling
),
496 weak_ptr_factory_(this) {
497 DVLOG(1) << "Created WebContentsCaptureMachine for "
498 << render_process_id
<< ':' << main_render_frame_id
499 << (auto_throttling_enabled_
? " with auto-throttling enabled" : "");
502 WebContentsCaptureMachine::~WebContentsCaptureMachine() {}
504 bool WebContentsCaptureMachine::IsStarted() const {
505 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
506 return weak_ptr_factory_
.HasWeakPtrs();
509 void WebContentsCaptureMachine::Start(
510 const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle_proxy
,
511 const media::VideoCaptureParams
& params
,
512 const base::Callback
<void(bool)> callback
) {
513 // Starts the capture machine asynchronously.
514 BrowserThread::PostTaskAndReplyWithResult(
517 base::Bind(&WebContentsCaptureMachine::InternalStart
,
518 base::Unretained(this),
524 bool WebContentsCaptureMachine::InternalStart(
525 const scoped_refptr
<media::ThreadSafeCaptureOracle
>& oracle_proxy
,
526 const media::VideoCaptureParams
& params
) {
527 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
528 DCHECK(!IsStarted());
530 DCHECK(oracle_proxy
.get());
531 oracle_proxy_
= oracle_proxy
;
532 capture_params_
= params
;
534 render_thread_
.reset(new base::Thread("WebContentsVideo_RenderThread"));
535 if (!render_thread_
->Start()) {
536 DVLOG(1) << "Failed to spawn render thread.";
537 render_thread_
.reset();
541 // Note: Creation of the first WeakPtr in the following statement will cause
542 // IsStarted() to return true from now on.
543 tracker_
->SetResizeChangeCallback(
544 base::Bind(&WebContentsCaptureMachine::UpdateCaptureSize
,
545 weak_ptr_factory_
.GetWeakPtr()));
546 tracker_
->Start(initial_render_process_id_
, initial_main_render_frame_id_
,
547 base::Bind(&WebContentsCaptureMachine::RenewFrameSubscription
,
548 weak_ptr_factory_
.GetWeakPtr()));
553 void WebContentsCaptureMachine::Stop(const base::Closure
& callback
) {
554 // Stops the capture machine asynchronously.
555 BrowserThread::PostTask(
556 BrowserThread::UI
, FROM_HERE
, base::Bind(
557 &WebContentsCaptureMachine::InternalStop
,
558 base::Unretained(this),
562 void WebContentsCaptureMachine::InternalStop(const base::Closure
& callback
) {
563 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
570 // The following cancels any outstanding callbacks and causes IsStarted() to
571 // return false from here onward.
572 weak_ptr_factory_
.InvalidateWeakPtrs();
574 // Note: RenewFrameSubscription() must be called before stopping |tracker_| so
575 // the web_contents() can be notified that the capturing is ending.
576 RenewFrameSubscription(false);
579 // The render thread cannot be stopped on the UI thread, so post a message
580 // to the thread pool used for blocking operations.
581 if (render_thread_
.get()) {
582 BrowserThread::PostBlockingPoolTask(
584 base::Bind(&DeleteOnWorkerThread
, base::Passed(&render_thread_
),
589 void WebContentsCaptureMachine::Capture(
590 const base::TimeTicks
& start_time
,
591 const scoped_refptr
<media::VideoFrame
>& target
,
592 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&
594 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
596 RenderWidgetHost
* rwh
= tracker_
->GetTargetRenderWidgetHost();
597 RenderWidgetHostViewBase
* view
=
598 rwh
? static_cast<RenderWidgetHostViewBase
*>(rwh
->GetView()) : NULL
;
600 deliver_frame_cb
.Run(base::TimeTicks(), false);
604 gfx::Size view_size
= view
->GetViewBounds().size();
605 if (view_size
!= last_view_size_
) {
606 last_view_size_
= view_size
;
608 // Measure the number of kilopixels.
609 UMA_HISTOGRAM_COUNTS_10000(
610 "TabCapture.ViewChangeKiloPixels",
611 view_size
.width() * view_size
.height() / 1024);
614 if (view
->CanCopyToVideoFrame()) {
615 view
->CopyFromCompositingSurfaceToVideoFrame(
616 gfx::Rect(view_size
),
618 base::Bind(&WebContentsCaptureMachine::
619 DidCopyFromCompositingSurfaceToVideoFrame
,
620 weak_ptr_factory_
.GetWeakPtr(),
621 start_time
, deliver_frame_cb
));
623 const gfx::Size fitted_size
= view_size
.IsEmpty() ? gfx::Size() :
624 media::ComputeLetterboxRegion(target
->visible_rect(), view_size
).size();
625 rwh
->CopyFromBackingStore(
627 fitted_size
, // Size here is a request not always honored.
628 base::Bind(&WebContentsCaptureMachine::DidCopyFromBackingStore
,
629 weak_ptr_factory_
.GetWeakPtr(),
637 gfx::Size
WebContentsCaptureMachine::ComputeOptimalViewSize() const {
638 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
640 // TODO(miu): Propagate capture frame size changes as new "preferred size"
641 // updates, rather than just using the max frame size.
642 // http://crbug.com/350491
643 gfx::Size optimal_size
= oracle_proxy_
->max_frame_size();
645 switch (capture_params_
.resolution_change_policy
) {
646 case media::RESOLUTION_POLICY_FIXED_RESOLUTION
:
648 case media::RESOLUTION_POLICY_FIXED_ASPECT_RATIO
:
649 case media::RESOLUTION_POLICY_ANY_WITHIN_LIMIT
: {
650 // If the max frame size is close to a common video aspect ratio, compute
651 // a standard resolution for that aspect ratio. For example, given
652 // 1365x768, which is very close to 16:9, the optimal size would be
653 // 1280x720. The purpose of this logic is to prevent scaling quality
654 // issues caused by "one pixel stretching" and/or odd-to-even dimension
655 // scaling, and to improve the performance of consumers of the captured
657 const auto HasIntendedAspectRatio
=
658 [](const gfx::Size
& size
, int width_units
, int height_units
) {
659 const int a
= height_units
* size
.width();
660 const int b
= width_units
* size
.height();
661 const int percentage_diff
= 100 * std::abs((a
- b
)) / b
;
662 return percentage_diff
<= 1; // Effectively, anything strictly <2%.
664 const auto RoundToExactAspectRatio
=
665 [](const gfx::Size
& size
, int width_step
, int height_step
) {
666 const int adjusted_height
=
667 std::max(size
.height() - (size
.height() % height_step
),
669 DCHECK_EQ((adjusted_height
* width_step
) % height_step
, 0);
670 return gfx::Size(adjusted_height
* width_step
/ height_step
,
673 if (HasIntendedAspectRatio(optimal_size
, 16, 9))
674 optimal_size
= RoundToExactAspectRatio(optimal_size
, 160, 90);
675 else if (HasIntendedAspectRatio(optimal_size
, 4, 3))
676 optimal_size
= RoundToExactAspectRatio(optimal_size
, 64, 48);
677 // Else, do not make an adjustment.
682 // If the ratio between physical and logical pixels is greater than 1:1,
683 // shrink |optimal_size| by that amount. Then, when external code resizes the
684 // render widget to the "preferred size," the widget will be physically
685 // rendered at the exact capture size, thereby eliminating unnecessary scaling
686 // operations in the graphics pipeline.
687 RenderWidgetHost
* const rwh
= tracker_
->GetTargetRenderWidgetHost();
688 RenderWidgetHostView
* const rwhv
= rwh
? rwh
->GetView() : NULL
;
690 const gfx::NativeView view
= rwhv
->GetNativeView();
691 const float scale
= ui::GetScaleFactorForNativeView(view
);
693 const gfx::Size
shrunk_size(
694 gfx::ToFlooredSize(gfx::ScaleSize(optimal_size
, 1.0f
/ scale
)));
695 if (shrunk_size
.width() > 0 && shrunk_size
.height() > 0)
696 optimal_size
= shrunk_size
;
700 VLOG(1) << "Computed optimal target size: " << optimal_size
.ToString();
704 void WebContentsCaptureMachine::DidCopyFromBackingStore(
705 const base::TimeTicks
& start_time
,
706 const scoped_refptr
<media::VideoFrame
>& target
,
707 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&
709 const SkBitmap
& bitmap
,
710 ReadbackResponse response
) {
711 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
713 base::TimeTicks now
= base::TimeTicks::Now();
714 DCHECK(render_thread_
.get());
715 if (response
== READBACK_SUCCESS
) {
716 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now
- start_time
);
717 TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target
.get(),
719 render_thread_
->task_runner()->PostTask(
720 FROM_HERE
, base::Bind(&RenderVideoFrame
, bitmap
, target
,
721 base::Bind(deliver_frame_cb
, start_time
)));
723 // Capture can fail due to transient issues, so just skip this frame.
724 DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
725 deliver_frame_cb
.Run(start_time
, false);
729 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
730 const base::TimeTicks
& start_time
,
731 const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback
&
734 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
735 base::TimeTicks now
= base::TimeTicks::Now();
738 UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now
- start_time
);
740 // Capture can fail due to transient issues, so just skip this frame.
741 DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
743 deliver_frame_cb
.Run(start_time
, success
);
746 void WebContentsCaptureMachine::RenewFrameSubscription(bool had_target
) {
747 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
749 RenderWidgetHost
* const rwh
=
750 had_target
? tracker_
->GetTargetRenderWidgetHost() : nullptr;
752 // Always destroy the old subscription before creating a new one.
753 const bool had_subscription
= !!subscription_
;
754 subscription_
.reset();
756 DVLOG(1) << "Renewing frame subscription to RWH@" << rwh
757 << ", had_subscription=" << had_subscription
;
760 if (had_subscription
&& tracker_
->web_contents())
761 tracker_
->web_contents()->DecrementCapturerCount();
763 // Tracking of WebContents and/or its main frame has failed before Stop()
764 // was called, so report this as an error:
765 oracle_proxy_
->ReportError("WebContents and/or main frame are gone.");
770 if (!had_subscription
&& tracker_
->web_contents())
771 tracker_
->web_contents()->IncrementCapturerCount(ComputeOptimalViewSize());
773 subscription_
.reset(new ContentCaptureSubscription(*rwh
, oracle_proxy_
,
774 base::Bind(&WebContentsCaptureMachine::Capture
,
775 weak_ptr_factory_
.GetWeakPtr())));
778 void WebContentsCaptureMachine::UpdateCaptureSize() {
779 DCHECK_CURRENTLY_ON(BrowserThread::UI
);
783 RenderWidgetHost
* const rwh
= tracker_
->GetTargetRenderWidgetHost();
784 RenderWidgetHostView
* const view
= rwh
? rwh
->GetView() : nullptr;
788 // Convert the view's size from the DIP coordinate space to the pixel
789 // coordinate space. When the view is being rendered on a high-DPI display,
790 // this allows the high-resolution image detail to propagate through to the
792 const gfx::Size view_size
= view
->GetViewBounds().size();
793 const gfx::Size physical_size
= gfx::ConvertSizeToPixel(
794 ui::GetScaleFactorForNativeView(view
->GetNativeView()), view_size
);
795 VLOG(1) << "Computed physical capture size (" << physical_size
.ToString()
796 << ") from view size (" << view_size
.ToString() << ").";
798 oracle_proxy_
->UpdateCaptureSize(physical_size
);
803 WebContentsVideoCaptureDevice::WebContentsVideoCaptureDevice(
804 int render_process_id
,
805 int main_render_frame_id
,
806 bool enable_auto_throttling
)
807 : core_(new media::ScreenCaptureDeviceCore(
808 scoped_ptr
<media::VideoCaptureMachine
>(new WebContentsCaptureMachine(
810 main_render_frame_id
,
811 enable_auto_throttling
)))) {}
813 WebContentsVideoCaptureDevice::~WebContentsVideoCaptureDevice() {
814 DVLOG(2) << "WebContentsVideoCaptureDevice@" << this << " destroying.";
818 media::VideoCaptureDevice
* WebContentsVideoCaptureDevice::Create(
819 const std::string
& device_id
) {
820 // Parse device_id into render_process_id and main_render_frame_id.
821 int render_process_id
= -1;
822 int main_render_frame_id
= -1;
823 if (!WebContentsCaptureUtil::ExtractTabCaptureTarget(
824 device_id
, &render_process_id
, &main_render_frame_id
)) {
828 return new WebContentsVideoCaptureDevice(
830 main_render_frame_id
,
831 WebContentsCaptureUtil::IsAutoThrottlingOptionSet(device_id
));
834 void WebContentsVideoCaptureDevice::AllocateAndStart(
835 const media::VideoCaptureParams
& params
,
836 scoped_ptr
<Client
> client
) {
837 DVLOG(1) << "Allocating " << params
.requested_format
.frame_size
.ToString();
838 core_
->AllocateAndStart(params
, client
.Pass());
841 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
842 core_
->StopAndDeAllocate();
845 } // namespace content