content/browser/media/capture/desktop_capture_device_aura.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/desktop_capture_device_aura.h"
   6
   7 #include "base/logging.h"
   8 #include "base/timer/timer.h"
   9 #include "cc/output/copy_output_request.h"
  10 #include "cc/output/copy_output_result.h"
  11 #include "content/browser/compositor/image_transport_factory.h"
  12 #include "content/browser/media/capture/content_video_capture_device_core.h"
  13 #include "content/common/gpu/client/gl_helper.h"
  14 #include "content/public/browser/browser_thread.h"
  15 #include "media/base/video_util.h"
  16 #include "media/video/capture/video_capture_types.h"
  17 #include "skia/ext/image_operations.h"
  18 #include "third_party/skia/include/core/SkBitmap.h"
  19 #include "ui/aura/env.h"
  20 #include "ui/aura/window.h"
  21 #include "ui/aura/window_observer.h"
  22 #include "ui/aura/window_tree_host.h"
  23 #include "ui/base/cursor/cursors_aura.h"
  24 #include "ui/compositor/compositor.h"
  25 #include "ui/compositor/dip_util.h"
  26 #include "ui/compositor/layer.h"
  27 #include "ui/gfx/screen.h"
  28
  29 namespace content {
  30
  31 namespace {
  32
  33 int clip_byte(int x) {
  34   return std::max(0, std::min(x, 255));
  35 }
  36
  37 int alpha_blend(int alpha, int src, int dst) {
  38   return (src * alpha + dst * (255 - alpha)) / 255;
  39 }
  40
  41 // Helper function to composite a cursor bitmap on a YUV420 video frame.
  42 void RenderCursorOnVideoFrame(
  43     const scoped_refptr<media::VideoFrame>& target,
  44     const SkBitmap& cursor_bitmap,
  45     const gfx::Point& cursor_position) {
  46   DCHECK(target);
  47   DCHECK(!cursor_bitmap.isNull());
  48
  49   gfx::Rect rect = gfx::IntersectRects(
  50       gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
  51           gfx::Vector2d(cursor_position.x(), cursor_position.y()),
  52       target->visible_rect());
  53
  54   cursor_bitmap.lockPixels();
  55   for (int y = rect.y(); y < rect.bottom(); ++y) {
  56     int cursor_y = y - cursor_position.y();
  57     uint8* yplane = target->data(media::VideoFrame::kYPlane) +
  58         y * target->row_bytes(media::VideoFrame::kYPlane);
  59     uint8* uplane = target->data(media::VideoFrame::kUPlane) +
  60         (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
  61     uint8* vplane = target->data(media::VideoFrame::kVPlane) +
  62         (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
  63     for (int x = rect.x(); x < rect.right(); ++x) {
  64       int cursor_x = x - cursor_position.x();
  65       SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
  66       int alpha = SkColorGetA(color);
  67       int color_r = SkColorGetR(color);
  68       int color_g = SkColorGetG(color);
  69       int color_b = SkColorGetB(color);
  70       int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
  71                                 128) >> 8) + 16);
  72       yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
  73
  74       // Only sample U and V at even coordinates.
  75       if ((x % 2 == 0) && (y % 2 == 0)) {
  76         int color_u = clip_byte(((color_r * -38 + color_g * -74 +
  77                                   color_b * 112 + 128) >> 8) + 128);
  78         int color_v = clip_byte(((color_r * 112 + color_g * -94 +
  79                                   color_b * -18 + 128) >> 8) + 128);
  80         uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
  81         vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
  82       }
  83     }
  84   }
  85   cursor_bitmap.unlockPixels();
  86 }
  87
  88 class DesktopVideoCaptureMachine
  89     : public VideoCaptureMachine,
  90       public aura::WindowObserver,
  91       public ui::CompositorObserver,
  92       public base::SupportsWeakPtr<DesktopVideoCaptureMachine> {
  93  public:
  94   DesktopVideoCaptureMachine(const DesktopMediaID& source);
  95   virtual ~DesktopVideoCaptureMachine();
  96
  97   // VideoCaptureFrameSource overrides.
  98   virtual bool Start(
  99       const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) OVERRIDE;
 100   virtual void Stop(const base::Closure& callback) OVERRIDE;
 101
 102   // Implements aura::WindowObserver.
 103   virtual void OnWindowBoundsChanged(aura::Window* window,
 104                                      const gfx::Rect& old_bounds,
 105                                      const gfx::Rect& new_bounds) OVERRIDE;
 106   virtual void OnWindowDestroyed(aura::Window* window) OVERRIDE;
 107
 108   // Implements ui::CompositorObserver.
 109   virtual void OnCompositingDidCommit(ui::Compositor* compositor) OVERRIDE {}
 110   virtual void OnCompositingStarted(ui::Compositor* compositor,
 111                                     base::TimeTicks start_time) OVERRIDE {}
 112   virtual void OnCompositingEnded(ui::Compositor* compositor) OVERRIDE;
 113   virtual void OnCompositingAborted(ui::Compositor* compositor) OVERRIDE {}
 114   virtual void OnCompositingLockStateChanged(
 115       ui::Compositor* compositor) OVERRIDE {}
 116
 117  private:
 118   // Captures a frame.
 119   // |dirty| is false for timer polls and true for compositor updates.
 120   void Capture(bool dirty);
 121
 122   // Update capture size. Must be called on the UI thread.
 123   void UpdateCaptureSize();
 124
 125   // Response callback for cc::Layer::RequestCopyOfOutput().
 126   void DidCopyOutput(
 127       scoped_refptr<media::VideoFrame> video_frame,
 128       base::TimeTicks start_time,
 129       const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 130       scoped_ptr<cc::CopyOutputResult> result);
 131
 132   // Helper function to update cursor state.
 133   // |region_in_frame| defines the desktop bound in the captured frame.
 134   // Returns the current cursor position in captured frame.
 135   gfx::Point UpdateCursorState(const gfx::Rect& region_in_frame);
 136
 137   // Clears cursor state.
 138   void ClearCursorState();
 139
 140   // The window associated with the desktop.
 141   aura::Window* desktop_window_;
 142
 143   // The layer associated with the desktop.
 144   ui::Layer* desktop_layer_;
 145
 146   // The timer that kicks off period captures.
 147   base::Timer timer_;
 148
 149   // The id of the window being captured.
 150   DesktopMediaID window_id_;
 151
 152   // Makes all the decisions about which frames to copy, and how.
 153   scoped_refptr<ThreadSafeCaptureOracle> oracle_proxy_;
 154
 155   // YUV readback pipeline.
 156   scoped_ptr<content::ReadbackYUVInterface> yuv_readback_pipeline_;
 157
 158   // Cursor state.
 159   ui::Cursor last_cursor_;
 160   gfx::Point cursor_hot_point_;
 161   SkBitmap scaled_cursor_bitmap_;
 162
 163   DISALLOW_COPY_AND_ASSIGN(DesktopVideoCaptureMachine);
 164 };
 165
 166 DesktopVideoCaptureMachine::DesktopVideoCaptureMachine(
 167     const DesktopMediaID& source)
 168     : desktop_window_(NULL),
 169       desktop_layer_(NULL),
 170       timer_(true, true),
 171       window_id_(source) {}
 172
 173 DesktopVideoCaptureMachine::~DesktopVideoCaptureMachine() {}
 174
 175 bool DesktopVideoCaptureMachine::Start(
 176     const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy) {
 177   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 178
 179   desktop_window_ = content::DesktopMediaID::GetAuraWindowById(window_id_);
 180   if (!desktop_window_)
 181     return false;
 182
 183   // If the desktop layer is already destroyed then return failure.
 184   desktop_layer_ = desktop_window_->layer();
 185   if (!desktop_layer_)
 186     return false;
 187
 188   DCHECK(oracle_proxy.get());
 189   oracle_proxy_ = oracle_proxy;
 190
 191   // Update capture size.
 192   UpdateCaptureSize();
 193
 194   // Start observing window events.
 195   desktop_window_->AddObserver(this);
 196
 197   // Start observing compositor updates.
 198   ui::Compositor* compositor = desktop_layer_->GetCompositor();
 199   if (!compositor)
 200     return false;
 201
 202   compositor->AddObserver(this);
 203
 204   // Starts timer.
 205   timer_.Start(FROM_HERE, oracle_proxy_->capture_period(),
 206                base::Bind(&DesktopVideoCaptureMachine::Capture, AsWeakPtr(),
 207                           false));
 208
 209   started_ = true;
 210   return true;
 211 }
 212
 213 void DesktopVideoCaptureMachine::Stop(const base::Closure& callback) {
 214   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 215
 216   // Stop observing window events.
 217   if (desktop_window_) {
 218     desktop_window_->RemoveObserver(this);
 219     desktop_window_ = NULL;
 220   }
 221
 222   // Stop observing compositor updates.
 223   if (desktop_layer_) {
 224     ui::Compositor* compositor = desktop_layer_->GetCompositor();
 225     if (compositor)
 226       compositor->RemoveObserver(this);
 227     desktop_layer_ = NULL;
 228   }
 229
 230   // Stop timer.
 231   timer_.Stop();
 232
 233   started_ = false;
 234
 235   callback.Run();
 236 }
 237
 238 void DesktopVideoCaptureMachine::UpdateCaptureSize() {
 239   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 240   if (oracle_proxy_ && desktop_layer_) {
 241     oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel(
 242         desktop_layer_, desktop_layer_->bounds().size()));
 243   }
 244   ClearCursorState();
 245 }
 246
 247 void DesktopVideoCaptureMachine::Capture(bool dirty) {
 248   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 249
 250   // Do not capture if the desktop layer is already destroyed.
 251   if (!desktop_layer_)
 252     return;
 253
 254   scoped_refptr<media::VideoFrame> frame;
 255   ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
 256
 257   const base::TimeTicks start_time = base::TimeTicks::Now();
 258   const VideoCaptureOracle::Event event =
 259       dirty ? VideoCaptureOracle::kCompositorUpdate
 260             : VideoCaptureOracle::kTimerPoll;
 261   if (oracle_proxy_->ObserveEventAndDecideCapture(
 262       event, start_time, &frame, &capture_frame_cb)) {
 263     scoped_ptr<cc::CopyOutputRequest> request =
 264         cc::CopyOutputRequest::CreateRequest(
 265             base::Bind(&DesktopVideoCaptureMachine::DidCopyOutput,
 266                        AsWeakPtr(), frame, start_time, capture_frame_cb));
 267     gfx::Rect window_rect =
 268         ui::ConvertRectToPixel(desktop_window_->layer(),
 269                                gfx::Rect(desktop_window_->bounds().width(),
 270                                          desktop_window_->bounds().height()));
 271     request->set_area(window_rect);
 272     desktop_layer_->RequestCopyOfOutput(request.Pass());
 273   }
 274 }
 275
 276 void CopyOutputFinishedForVideo(
 277     base::TimeTicks start_time,
 278     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 279     const scoped_refptr<media::VideoFrame>& target,
 280     const SkBitmap& cursor_bitmap,
 281     const gfx::Point& cursor_position,
 282     scoped_ptr<cc::SingleReleaseCallback> release_callback,
 283     bool result) {
 284   if (!cursor_bitmap.isNull())
 285     RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
 286   release_callback->Run(0, false);
 287   capture_frame_cb.Run(start_time, result);
 288 }
 289
 290 void DesktopVideoCaptureMachine::DidCopyOutput(
 291     scoped_refptr<media::VideoFrame> video_frame,
 292     base::TimeTicks start_time,
 293     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 294     scoped_ptr<cc::CopyOutputResult> result) {
 295   if (result->IsEmpty() || result->size().IsEmpty() || !desktop_layer_)
 296     return;
 297
 298   // Compute the dest size we want after the letterboxing resize. Make the
 299   // coordinates and sizes even because we letterbox in YUV space
 300   // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
 301   // line up correctly.
 302   // The video frame's coded_size() and the result's size() are both physical
 303   // pixels.
 304   gfx::Rect region_in_frame =
 305       media::ComputeLetterboxRegion(gfx::Rect(video_frame->coded_size()),
 306                                     result->size());
 307   region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
 308                               region_in_frame.y() & ~1,
 309                               region_in_frame.width() & ~1,
 310                               region_in_frame.height() & ~1);
 311   if (region_in_frame.IsEmpty())
 312     return;
 313
 314   ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
 315   GLHelper* gl_helper = factory->GetGLHelper();
 316   if (!gl_helper)
 317     return;
 318
 319   cc::TextureMailbox texture_mailbox;
 320   scoped_ptr<cc::SingleReleaseCallback> release_callback;
 321   result->TakeTexture(&texture_mailbox, &release_callback);
 322   DCHECK(texture_mailbox.IsTexture());
 323   if (!texture_mailbox.IsTexture())
 324     return;
 325
 326   gfx::Rect result_rect(result->size());
 327   if (!yuv_readback_pipeline_ ||
 328       yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
 329       yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
 330       yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
 331     yuv_readback_pipeline_.reset(
 332         gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
 333                                              result_rect.size(),
 334                                              result_rect,
 335                                              video_frame->coded_size(),
 336                                              region_in_frame,
 337                                              true,
 338                                              true));
 339   }
 340
 341   gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
 342   yuv_readback_pipeline_->ReadbackYUV(
 343       texture_mailbox.mailbox(),
 344       texture_mailbox.sync_point(),
 345       video_frame.get(),
 346       base::Bind(&CopyOutputFinishedForVideo,
 347                  start_time,
 348                  capture_frame_cb,
 349                  video_frame,
 350                  scaled_cursor_bitmap_,
 351                  cursor_position_in_frame,
 352                  base::Passed(&release_callback)));
 353 }
 354
 355 gfx::Point DesktopVideoCaptureMachine::UpdateCursorState(
 356     const gfx::Rect& region_in_frame) {
 357   const gfx::Rect desktop_bounds = desktop_layer_->bounds();
 358   gfx::NativeCursor cursor =
 359       desktop_window_->GetHost()->last_cursor();
 360   if (last_cursor_ != cursor) {
 361     SkBitmap cursor_bitmap;
 362     if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
 363       scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
 364           cursor_bitmap,
 365           skia::ImageOperations::RESIZE_BEST,
 366           cursor_bitmap.width() * region_in_frame.width() /
 367               desktop_bounds.width(),
 368           cursor_bitmap.height() * region_in_frame.height() /
 369               desktop_bounds.height());
 370       last_cursor_ = cursor;
 371     } else {
 372       // Clear cursor state if ui::GetCursorBitmap failed so that we do not
 373       // render cursor on the captured frame.
 374       ClearCursorState();
 375     }
 376   }
 377
 378   gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
 379   const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
 380       desktop_layer_, cursor_hot_point_);
 381   cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
 382                          -desktop_bounds.y() - hot_point_in_dip.y());
 383   return gfx::Point(
 384       region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
 385           desktop_bounds.width(),
 386       region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
 387           desktop_bounds.height());
 388 }
 389
 390 void DesktopVideoCaptureMachine::ClearCursorState() {
 391   last_cursor_ = ui::Cursor();
 392   cursor_hot_point_ = gfx::Point();
 393   scaled_cursor_bitmap_.reset();
 394 }
 395
 396 void DesktopVideoCaptureMachine::OnWindowBoundsChanged(
 397     aura::Window* window,
 398     const gfx::Rect& old_bounds,
 399     const gfx::Rect& new_bounds) {
 400   DCHECK(desktop_window_ && window == desktop_window_);
 401
 402   // Post task to update capture size on UI thread.
 403   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 404       &DesktopVideoCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
 405 }
 406
 407 void DesktopVideoCaptureMachine::OnWindowDestroyed(aura::Window* window) {
 408   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
 409
 410   Stop(base::Bind(&base::DoNothing));
 411
 412   oracle_proxy_->ReportError("OnWindowDestroyed()");
 413 }
 414
 415 void DesktopVideoCaptureMachine::OnCompositingEnded(
 416     ui::Compositor* compositor) {
 417   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 418       &DesktopVideoCaptureMachine::Capture, AsWeakPtr(), true));
 419 }
 420
 421 }  // namespace
 422
 423 DesktopCaptureDeviceAura::DesktopCaptureDeviceAura(
 424     const DesktopMediaID& source)
 425     : core_(new ContentVideoCaptureDeviceCore(scoped_ptr<VideoCaptureMachine>(
 426         new DesktopVideoCaptureMachine(source)))) {}
 427
 428 DesktopCaptureDeviceAura::~DesktopCaptureDeviceAura() {
 429   DVLOG(2) << "DesktopCaptureDeviceAura@" << this << " destroying.";
 430 }
 431
 432 // static
 433 media::VideoCaptureDevice* DesktopCaptureDeviceAura::Create(
 434     const DesktopMediaID& source) {
 435   return new DesktopCaptureDeviceAura(source);
 436 }
 437
 438 void DesktopCaptureDeviceAura::AllocateAndStart(
 439     const media::VideoCaptureParams& params,
 440     scoped_ptr<Client> client) {
 441   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
 442   core_->AllocateAndStart(params, client.Pass());
 443 }
 444
 445 void DesktopCaptureDeviceAura::StopAndDeAllocate() {
 446   core_->StopAndDeAllocate();
 447 }
 448
 449 }  // namespace content