1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "content/browser/media/capture/desktop_capture_device_aura.h"
7 #include "base/logging.h"
8 #include "base/metrics/histogram.h"
9 #include "base/timer/timer.h"
10 #include "cc/output/copy_output_request.h"
11 #include "cc/output/copy_output_result.h"
12 #include "content/browser/compositor/image_transport_factory.h"
13 #include "content/browser/media/capture/content_video_capture_device_core.h"
14 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
15 #include "content/common/gpu/client/gl_helper.h"
16 #include "content/public/browser/browser_thread.h"
17 #include "media/base/bind_to_current_loop.h"
18 #include "media/base/video_util.h"
19 #include "media/video/capture/video_capture_types.h"
20 #include "skia/ext/image_operations.h"
21 #include "third_party/skia/include/core/SkBitmap.h"
22 #include "ui/aura/client/screen_position_client.h"
23 #include "ui/aura/env.h"
24 #include "ui/aura/window.h"
25 #include "ui/aura/window_observer.h"
26 #include "ui/aura/window_tree_host.h"
27 #include "ui/base/cursor/cursors_aura.h"
28 #include "ui/compositor/compositor.h"
29 #include "ui/compositor/dip_util.h"
30 #include "ui/compositor/layer.h"
31 #include "ui/gfx/screen.h"
37 int clip_byte(int x
) {
38 return std::max(0, std::min(x
, 255));
41 int alpha_blend(int alpha
, int src
, int dst
) {
42 return (src
* alpha
+ dst
* (255 - alpha
)) / 255;
45 // Helper function to composite a cursor bitmap on a YUV420 video frame.
46 void RenderCursorOnVideoFrame(
47 const scoped_refptr
<media::VideoFrame
>& target
,
48 const SkBitmap
& cursor_bitmap
,
49 const gfx::Point
& cursor_position
) {
51 DCHECK(!cursor_bitmap
.isNull());
53 gfx::Rect rect
= gfx::IntersectRects(
54 gfx::Rect(cursor_bitmap
.width(), cursor_bitmap
.height()) +
55 gfx::Vector2d(cursor_position
.x(), cursor_position
.y()),
56 target
->visible_rect());
58 cursor_bitmap
.lockPixels();
59 for (int y
= rect
.y(); y
< rect
.bottom(); ++y
) {
60 int cursor_y
= y
- cursor_position
.y();
61 uint8
* yplane
= target
->data(media::VideoFrame::kYPlane
) +
62 y
* target
->row_bytes(media::VideoFrame::kYPlane
);
63 uint8
* uplane
= target
->data(media::VideoFrame::kUPlane
) +
64 (y
/ 2) * target
->row_bytes(media::VideoFrame::kUPlane
);
65 uint8
* vplane
= target
->data(media::VideoFrame::kVPlane
) +
66 (y
/ 2) * target
->row_bytes(media::VideoFrame::kVPlane
);
67 for (int x
= rect
.x(); x
< rect
.right(); ++x
) {
68 int cursor_x
= x
- cursor_position
.x();
69 SkColor color
= cursor_bitmap
.getColor(cursor_x
, cursor_y
);
70 int alpha
= SkColorGetA(color
);
71 int color_r
= SkColorGetR(color
);
72 int color_g
= SkColorGetG(color
);
73 int color_b
= SkColorGetB(color
);
74 int color_y
= clip_byte(((color_r
* 66 + color_g
* 129 + color_b
* 25 +
76 yplane
[x
] = alpha_blend(alpha
, color_y
, yplane
[x
]);
78 // Only sample U and V at even coordinates.
79 if ((x
% 2 == 0) && (y
% 2 == 0)) {
80 int color_u
= clip_byte(((color_r
* -38 + color_g
* -74 +
81 color_b
* 112 + 128) >> 8) + 128);
82 int color_v
= clip_byte(((color_r
* 112 + color_g
* -94 +
83 color_b
* -18 + 128) >> 8) + 128);
84 uplane
[x
/ 2] = alpha_blend(alpha
, color_u
, uplane
[x
/ 2]);
85 vplane
[x
/ 2] = alpha_blend(alpha
, color_v
, vplane
[x
/ 2]);
89 cursor_bitmap
.unlockPixels();
92 class DesktopVideoCaptureMachine
93 : public VideoCaptureMachine
,
94 public aura::WindowObserver
,
95 public ui::CompositorObserver
,
96 public base::SupportsWeakPtr
<DesktopVideoCaptureMachine
> {
98 DesktopVideoCaptureMachine(const DesktopMediaID
& source
);
99 virtual ~DesktopVideoCaptureMachine();
101 // VideoCaptureFrameSource overrides.
102 virtual bool Start(const scoped_refptr
<ThreadSafeCaptureOracle
>& oracle_proxy
,
103 const media::VideoCaptureParams
& params
) OVERRIDE
;
104 virtual void Stop(const base::Closure
& callback
) OVERRIDE
;
106 // Implements aura::WindowObserver.
107 virtual void OnWindowBoundsChanged(aura::Window
* window
,
108 const gfx::Rect
& old_bounds
,
109 const gfx::Rect
& new_bounds
) OVERRIDE
;
110 virtual void OnWindowDestroyed(aura::Window
* window
) OVERRIDE
;
111 virtual void OnWindowAddedToRootWindow(aura::Window
* window
) OVERRIDE
;
112 virtual void OnWindowRemovingFromRootWindow(aura::Window
* window
,
113 aura::Window
* new_root
) OVERRIDE
;
115 // Implements ui::CompositorObserver.
116 virtual void OnCompositingDidCommit(ui::Compositor
* compositor
) OVERRIDE
{}
117 virtual void OnCompositingStarted(ui::Compositor
* compositor
,
118 base::TimeTicks start_time
) OVERRIDE
{}
119 virtual void OnCompositingEnded(ui::Compositor
* compositor
) OVERRIDE
;
120 virtual void OnCompositingAborted(ui::Compositor
* compositor
) OVERRIDE
{}
121 virtual void OnCompositingLockStateChanged(
122 ui::Compositor
* compositor
) OVERRIDE
{}
126 // |dirty| is false for timer polls and true for compositor updates.
127 void Capture(bool dirty
);
129 // Update capture size. Must be called on the UI thread.
130 void UpdateCaptureSize();
132 // Response callback for cc::Layer::RequestCopyOfOutput().
134 scoped_refptr
<media::VideoFrame
> video_frame
,
135 base::TimeTicks start_time
,
136 const ThreadSafeCaptureOracle::CaptureFrameCallback
& capture_frame_cb
,
137 scoped_ptr
<cc::CopyOutputResult
> result
);
139 // A helper which does the real work for DidCopyOutput. Returns true if
141 bool ProcessCopyOutputResponse(
142 scoped_refptr
<media::VideoFrame
> video_frame
,
143 base::TimeTicks start_time
,
144 const ThreadSafeCaptureOracle::CaptureFrameCallback
& capture_frame_cb
,
145 scoped_ptr
<cc::CopyOutputResult
> result
);
147 // Helper function to update cursor state.
148 // |region_in_frame| defines the desktop bound in the captured frame.
149 // Returns the current cursor position in captured frame.
150 gfx::Point
UpdateCursorState(const gfx::Rect
& region_in_frame
);
152 // Clears cursor state.
153 void ClearCursorState();
155 // The window associated with the desktop.
156 aura::Window
* desktop_window_
;
158 // The timer that kicks off period captures.
161 // The id of the window being captured.
162 DesktopMediaID window_id_
;
164 // Makes all the decisions about which frames to copy, and how.
165 scoped_refptr
<ThreadSafeCaptureOracle
> oracle_proxy_
;
167 // The capture parameters for this capture.
168 media::VideoCaptureParams capture_params_
;
170 // YUV readback pipeline.
171 scoped_ptr
<content::ReadbackYUVInterface
> yuv_readback_pipeline_
;
174 ui::Cursor last_cursor_
;
175 gfx::Point cursor_hot_point_
;
176 SkBitmap scaled_cursor_bitmap_
;
178 DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine
);
181 DesktopVideoCaptureMachine::DesktopVideoCaptureMachine(
182 const DesktopMediaID
& source
)
183 : desktop_window_(NULL
),
185 window_id_(source
) {}
187 DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {}
189 bool DesktopVideoCaptureMachine::Start(
190 const scoped_refptr
<ThreadSafeCaptureOracle
>& oracle_proxy
,
191 const media::VideoCaptureParams
& params
) {
192 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
194 desktop_window_
= content::DesktopMediaID::GetAuraWindowById(window_id_
);
195 if (!desktop_window_
)
198 // If the associated layer is already destroyed then return failure.
199 ui::Layer
* layer
= desktop_window_
->layer();
203 DCHECK(oracle_proxy
.get());
204 oracle_proxy_
= oracle_proxy
;
205 capture_params_
= params
;
207 // Update capture size.
210 // Start observing window events.
211 desktop_window_
->AddObserver(this);
213 // Start observing compositor updates.
214 if (desktop_window_
->GetHost())
215 desktop_window_
->GetHost()->compositor()->AddObserver(this);
218 timer_
.Start(FROM_HERE
, oracle_proxy_
->capture_period(),
219 base::Bind(&DesktopVideoCaptureMachine::Capture
, AsWeakPtr(),
226 void DesktopVideoCaptureMachine::Stop(const base::Closure
& callback
) {
227 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
229 // Stop observing compositor and window events.
230 if (desktop_window_
) {
231 if (desktop_window_
->GetHost())
232 desktop_window_
->GetHost()->compositor()->RemoveObserver(this);
233 desktop_window_
->RemoveObserver(this);
234 desktop_window_
= NULL
;
245 void DesktopVideoCaptureMachine::UpdateCaptureSize() {
246 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
247 if (oracle_proxy_
&& desktop_window_
) {
248 ui::Layer
* layer
= desktop_window_
->layer();
249 oracle_proxy_
->UpdateCaptureSize(ui::ConvertSizeToPixel(
250 layer
, layer
->bounds().size()));
255 void DesktopVideoCaptureMachine::Capture(bool dirty
) {
256 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
258 // Do not capture if the desktop window is already destroyed.
259 if (!desktop_window_
)
262 scoped_refptr
<media::VideoFrame
> frame
;
263 ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb
;
265 const base::TimeTicks start_time
= base::TimeTicks::Now();
266 const VideoCaptureOracle::Event event
=
267 dirty
? VideoCaptureOracle::kCompositorUpdate
268 : VideoCaptureOracle::kTimerPoll
;
269 if (oracle_proxy_
->ObserveEventAndDecideCapture(
270 event
, start_time
, &frame
, &capture_frame_cb
)) {
271 scoped_ptr
<cc::CopyOutputRequest
> request
=
272 cc::CopyOutputRequest::CreateRequest(
273 base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput
,
274 AsWeakPtr(), frame
, start_time
, capture_frame_cb
));
275 gfx::Rect window_rect
=
276 ui::ConvertRectToPixel(desktop_window_
->layer(),
277 gfx::Rect(desktop_window_
->bounds().width(),
278 desktop_window_
->bounds().height()));
279 request
->set_area(window_rect
);
280 desktop_window_
->layer()->RequestCopyOfOutput(request
.Pass());
284 void CopyOutputFinishedForVideo(
285 base::TimeTicks start_time
,
286 const ThreadSafeCaptureOracle::CaptureFrameCallback
& capture_frame_cb
,
287 const scoped_refptr
<media::VideoFrame
>& target
,
288 const SkBitmap
& cursor_bitmap
,
289 const gfx::Point
& cursor_position
,
290 scoped_ptr
<cc::SingleReleaseCallback
> release_callback
,
292 if (!cursor_bitmap
.isNull())
293 RenderCursorOnVideoFrame(target
, cursor_bitmap
, cursor_position
);
294 release_callback
->Run(0, false);
295 capture_frame_cb
.Run(target
, start_time
, result
);
298 void RunSingleReleaseCallback(scoped_ptr
<cc::SingleReleaseCallback
> cb
,
299 const std::vector
<uint32
>& sync_points
) {
300 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
301 GLHelper
* gl_helper
= ImageTransportFactory::GetInstance()->GetGLHelper();
303 for (unsigned i
= 0; i
< sync_points
.size(); i
++)
304 gl_helper
->WaitSyncPoint(sync_points
[i
]);
305 uint32 new_sync_point
= gl_helper
->InsertSyncPoint();
306 cb
->Run(new_sync_point
, false);
309 void DesktopVideoCaptureMachine::DidCopyOutput(
310 scoped_refptr
<media::VideoFrame
> video_frame
,
311 base::TimeTicks start_time
,
312 const ThreadSafeCaptureOracle::CaptureFrameCallback
& capture_frame_cb
,
313 scoped_ptr
<cc::CopyOutputResult
> result
) {
314 static bool first_call
= true;
316 bool succeeded
= ProcessCopyOutputResponse(
317 video_frame
, start_time
, capture_frame_cb
, result
.Pass());
319 base::TimeDelta capture_time
= base::TimeTicks::Now() - start_time
;
321 window_id_
.type
== DesktopMediaID::TYPE_SCREEN
? kUmaScreenCaptureTime
322 : kUmaWindowCaptureTime
,
327 if (window_id_
.type
== DesktopMediaID::TYPE_SCREEN
) {
328 IncrementDesktopCaptureCounter(succeeded
? FIRST_SCREEN_CAPTURE_SUCCEEDED
329 : FIRST_SCREEN_CAPTURE_FAILED
);
331 IncrementDesktopCaptureCounter(succeeded
332 ? FIRST_WINDOW_CAPTURE_SUCCEEDED
333 : FIRST_WINDOW_CAPTURE_FAILED
);
338 bool DesktopVideoCaptureMachine::ProcessCopyOutputResponse(
339 scoped_refptr
<media::VideoFrame
> video_frame
,
340 base::TimeTicks start_time
,
341 const ThreadSafeCaptureOracle::CaptureFrameCallback
& capture_frame_cb
,
342 scoped_ptr
<cc::CopyOutputResult
> result
) {
343 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
344 if (result
->IsEmpty() || result
->size().IsEmpty() || !desktop_window_
)
347 if (capture_params_
.requested_format
.pixel_format
==
348 media::PIXEL_FORMAT_TEXTURE
) {
349 DCHECK(!video_frame
);
350 cc::TextureMailbox texture_mailbox
;
351 scoped_ptr
<cc::SingleReleaseCallback
> release_callback
;
352 result
->TakeTexture(&texture_mailbox
, &release_callback
);
353 DCHECK(texture_mailbox
.IsTexture());
354 if (!texture_mailbox
.IsTexture())
356 video_frame
= media::VideoFrame::WrapNativeTexture(
357 make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox
.mailbox(),
358 texture_mailbox
.target(),
359 texture_mailbox
.sync_point())),
360 media::BindToCurrentLoop(base::Bind(&RunSingleReleaseCallback
,
361 base::Passed(&release_callback
))),
363 gfx::Rect(result
->size()),
366 media::VideoFrame::ReadPixelsCB());
367 capture_frame_cb
.Run(video_frame
, start_time
, true);
371 // Compute the dest size we want after the letterboxing resize. Make the
372 // coordinates and sizes even because we letterbox in YUV space
373 // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
374 // line up correctly.
375 // The video frame's coded_size() and the result's size() are both physical
377 gfx::Rect region_in_frame
=
378 media::ComputeLetterboxRegion(gfx::Rect(video_frame
->coded_size()),
380 region_in_frame
= gfx::Rect(region_in_frame
.x() & ~1,
381 region_in_frame
.y() & ~1,
382 region_in_frame
.width() & ~1,
383 region_in_frame
.height() & ~1);
384 if (region_in_frame
.IsEmpty())
387 ImageTransportFactory
* factory
= ImageTransportFactory::GetInstance();
388 GLHelper
* gl_helper
= factory
->GetGLHelper();
392 cc::TextureMailbox texture_mailbox
;
393 scoped_ptr
<cc::SingleReleaseCallback
> release_callback
;
394 result
->TakeTexture(&texture_mailbox
, &release_callback
);
395 DCHECK(texture_mailbox
.IsTexture());
396 if (!texture_mailbox
.IsTexture())
399 gfx::Rect
result_rect(result
->size());
400 if (!yuv_readback_pipeline_
||
401 yuv_readback_pipeline_
->scaler()->SrcSize() != result_rect
.size() ||
402 yuv_readback_pipeline_
->scaler()->SrcSubrect() != result_rect
||
403 yuv_readback_pipeline_
->scaler()->DstSize() != region_in_frame
.size()) {
404 yuv_readback_pipeline_
.reset(
405 gl_helper
->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST
,
408 video_frame
->coded_size(),
414 gfx::Point cursor_position_in_frame
= UpdateCursorState(region_in_frame
);
415 yuv_readback_pipeline_
->ReadbackYUV(
416 texture_mailbox
.mailbox(),
417 texture_mailbox
.sync_point(),
419 base::Bind(&CopyOutputFinishedForVideo
,
423 scaled_cursor_bitmap_
,
424 cursor_position_in_frame
,
425 base::Passed(&release_callback
)));
429 gfx::Point
DesktopVideoCaptureMachine::UpdateCursorState(
430 const gfx::Rect
& region_in_frame
) {
431 const gfx::Rect desktop_bounds
= desktop_window_
->layer()->bounds();
432 gfx::NativeCursor cursor
=
433 desktop_window_
->GetHost()->last_cursor();
434 if (last_cursor_
!= cursor
) {
435 SkBitmap cursor_bitmap
;
436 if (ui::GetCursorBitmap(cursor
, &cursor_bitmap
, &cursor_hot_point_
)) {
437 scaled_cursor_bitmap_
= skia::ImageOperations::Resize(
439 skia::ImageOperations::RESIZE_BEST
,
440 cursor_bitmap
.width() * region_in_frame
.width() /
441 desktop_bounds
.width(),
442 cursor_bitmap
.height() * region_in_frame
.height() /
443 desktop_bounds
.height());
444 last_cursor_
= cursor
;
446 // Clear cursor state if ui::GetCursorBitmap failed so that we do not
447 // render cursor on the captured frame.
452 gfx::Point cursor_position
= aura::Env::GetInstance()->last_mouse_location();
453 aura::client::GetScreenPositionClient(desktop_window_
->GetRootWindow())->
454 ConvertPointFromScreen(desktop_window_
, &cursor_position
);
455 const gfx::Point hot_point_in_dip
= ui::ConvertPointToDIP(
456 desktop_window_
->layer(), cursor_hot_point_
);
457 cursor_position
.Offset(-desktop_bounds
.x() - hot_point_in_dip
.x(),
458 -desktop_bounds
.y() - hot_point_in_dip
.y());
460 region_in_frame
.x() + cursor_position
.x() * region_in_frame
.width() /
461 desktop_bounds
.width(),
462 region_in_frame
.y() + cursor_position
.y() * region_in_frame
.height() /
463 desktop_bounds
.height());
466 void DesktopVideoCaptureMachine::ClearCursorState() {
467 last_cursor_
= ui::Cursor();
468 cursor_hot_point_
= gfx::Point();
469 scaled_cursor_bitmap_
.reset();
472 void DesktopVideoCaptureMachine::OnWindowBoundsChanged(
473 aura::Window
* window
,
474 const gfx::Rect
& old_bounds
,
475 const gfx::Rect
& new_bounds
) {
476 DCHECK(desktop_window_
&& window
== desktop_window_
);
478 // Post task to update capture size on UI thread.
479 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, base::Bind(
480 &DesktopVideoCaptureMachine::UpdateCaptureSize
, AsWeakPtr()));
483 void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window
* window
) {
484 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI
));
486 Stop(base::Bind(&base::DoNothing
));
488 oracle_proxy_
->ReportError("OnWindowDestroyed()");
491 void DesktopVideoCaptureMachine::OnWindowAddedToRootWindow(
492 aura::Window
* window
) {
493 DCHECK(window
== desktop_window_
);
494 window
->GetHost()->compositor()->AddObserver(this);
497 void DesktopVideoCaptureMachine::OnWindowRemovingFromRootWindow(
498 aura::Window
* window
,
499 aura::Window
* new_root
) {
500 DCHECK(window
== desktop_window_
);
501 window
->GetHost()->compositor()->RemoveObserver(this);
504 void DesktopVideoCaptureMachine::OnCompositingEnded(
505 ui::Compositor
* compositor
) {
506 BrowserThread::PostTask(BrowserThread::UI
, FROM_HERE
, base::Bind(
507 &DesktopVideoCaptureMachine::Capture
, AsWeakPtr(), true));
512 DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(
513 const DesktopMediaID
& source
)
514 : core_(new ContentVideoCaptureDeviceCore(scoped_ptr
<VideoCaptureMachine
>(
515 new DesktopVideoCaptureMachine(source
)))) {}
517 DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
518 DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying.";
522 media::VideoCaptureDevice
* DesktopCaptureDeviceAura::Create(
523 const DesktopMediaID
& source
) {
524 IncrementDesktopCaptureCounter(source
.type
== DesktopMediaID::TYPE_SCREEN
525 ? SCREEN_CAPTURER_CREATED
526 : WINDOW_CAPTURER_CREATED
);
527 return new DesktopCaptureDeviceAura(source
);
530 void DesktopCaptureDeviceAura::AllocateAndStart(
531 const media::VideoCaptureParams
& params
,
532 scoped_ptr
<Client
> client
) {
533 DVLOG(1) << "Allocating " << params
.requested_format
.frame_size
.ToString();
534 core_
->AllocateAndStart(params
, client
.Pass());
537 void DesktopCaptureDeviceAura::StopAndDeAllocate() {
538 core_
->StopAndDeAllocate();
541 } // namespace content