content/browser/media/capture/aura_window_capture_machine.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/aura_window_capture_machine.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/timer/timer.h"
  10 #include "cc/output/copy_output_request.h"
  11 #include "cc/output/copy_output_result.h"
  12 #include "content/browser/compositor/image_transport_factory.h"
  13 #include "content/browser/media/capture/content_video_capture_device_core.h"
  14 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  15 #include "content/common/gpu/client/gl_helper.h"
  16 #include "content/public/browser/browser_thread.h"
  17 #include "content/public/browser/power_save_blocker.h"
  18 #include "media/base/video_capture_types.h"
  19 #include "media/base/video_util.h"
  20 #include "skia/ext/image_operations.h"
  21 #include "third_party/skia/include/core/SkBitmap.h"
  22 #include "ui/aura/client/screen_position_client.h"
  23 #include "ui/aura/env.h"
  24 #include "ui/aura/window.h"
  25 #include "ui/aura/window_observer.h"
  26 #include "ui/aura/window_tree_host.h"
  27 #include "ui/base/cursor/cursors_aura.h"
  28 #include "ui/compositor/compositor.h"
  29 #include "ui/compositor/dip_util.h"
  30 #include "ui/compositor/layer.h"
  31 #include "ui/gfx/screen.h"
  32
  33 namespace content {
  34
  35 namespace {
  36
  37 int clip_byte(int x) {
  38   return std::max(0, std::min(x, 255));
  39 }
  40
  41 int alpha_blend(int alpha, int src, int dst) {
  42   return (src * alpha + dst * (255 - alpha)) / 255;
  43 }
  44
  45 // Helper function to composite a cursor bitmap on a YUV420 video frame.
  46 void RenderCursorOnVideoFrame(
  47     const scoped_refptr<media::VideoFrame>& target,
  48     const SkBitmap& cursor_bitmap,
  49     const gfx::Point& cursor_position) {
  50   DCHECK(target.get());
  51   DCHECK(!cursor_bitmap.isNull());
  52
  53   gfx::Rect rect = gfx::IntersectRects(
  54       gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
  55           gfx::Vector2d(cursor_position.x(), cursor_position.y()),
  56       target->visible_rect());
  57
  58   cursor_bitmap.lockPixels();
  59   for (int y = rect.y(); y < rect.bottom(); ++y) {
  60     int cursor_y = y - cursor_position.y();
  61     uint8* yplane = target->data(media::VideoFrame::kYPlane) +
  62         y * target->row_bytes(media::VideoFrame::kYPlane);
  63     uint8* uplane = target->data(media::VideoFrame::kUPlane) +
  64         (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
  65     uint8* vplane = target->data(media::VideoFrame::kVPlane) +
  66         (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
  67     for (int x = rect.x(); x < rect.right(); ++x) {
  68       int cursor_x = x - cursor_position.x();
  69       SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
  70       int alpha = SkColorGetA(color);
  71       int color_r = SkColorGetR(color);
  72       int color_g = SkColorGetG(color);
  73       int color_b = SkColorGetB(color);
  74       int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
  75                                 128) >> 8) + 16);
  76       yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
  77
  78       // Only sample U and V at even coordinates.
  79       if ((x % 2 == 0) && (y % 2 == 0)) {
  80         int color_u = clip_byte(((color_r * -38 + color_g * -74 +
  81                                   color_b * 112 + 128) >> 8) + 128);
  82         int color_v = clip_byte(((color_r * 112 + color_g * -94 +
  83                                   color_b * -18 + 128) >> 8) + 128);
  84         uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
  85         vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
  86       }
  87     }
  88   }
  89   cursor_bitmap.unlockPixels();
  90 }
  91
  92 void CopyOutputFinishedForVideo(
  93     base::TimeTicks start_time,
  94     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
  95     const scoped_refptr<media::VideoFrame>& target,
  96     const SkBitmap& cursor_bitmap,
  97     const gfx::Point& cursor_position,
  98     scoped_ptr<cc::SingleReleaseCallback> release_callback,
  99     bool result) {
 100   if (!cursor_bitmap.isNull())
 101     RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
 102   release_callback->Run(0, false);
 103   capture_frame_cb.Run(target, start_time, result);
 104 }
 105
 106 void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb,
 107                               uint32 sync_point) {
 108   cb->Run(sync_point, false);
 109 }
 110
 111 }  // namespace
 112
 113 AuraWindowCaptureMachine::AuraWindowCaptureMachine()
 114     : desktop_window_(NULL),
 115       timer_(true, true),
 116       screen_capture_(false) {}
 117
 118 AuraWindowCaptureMachine::~AuraWindowCaptureMachine() {}
 119
 120 bool AuraWindowCaptureMachine::Start(
 121     const scoped_refptr<ThreadSafeCaptureOracle>& oracle_proxy,
 122     const media::VideoCaptureParams& params) {
 123   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 124
 125   // The window might be destroyed between SetWindow() and Start().
 126   if (!desktop_window_)
 127     return false;
 128
 129   // If the associated layer is already destroyed then return failure.
 130   ui::Layer* layer = desktop_window_->layer();
 131   if (!layer)
 132     return false;
 133
 134   DCHECK(oracle_proxy.get());
 135   oracle_proxy_ = oracle_proxy;
 136   capture_params_ = params;
 137
 138   // Update capture size.
 139   UpdateCaptureSize();
 140
 141   // Start observing compositor updates.
 142   if (desktop_window_->GetHost())
 143     desktop_window_->GetHost()->compositor()->AddObserver(this);
 144
 145   power_save_blocker_.reset(
 146       PowerSaveBlocker::Create(
 147           PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
 148           PowerSaveBlocker::kReasonOther,
 149           "DesktopCaptureDevice is running").release());
 150
 151   // Starts timer.
 152   timer_.Start(FROM_HERE, oracle_proxy_->min_capture_period(),
 153                base::Bind(&AuraWindowCaptureMachine::Capture, AsWeakPtr(),
 154                           false));
 155
 156   return true;
 157 }
 158
 159 void AuraWindowCaptureMachine::Stop(const base::Closure& callback) {
 160   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 161   power_save_blocker_.reset();
 162
 163   // Stop observing compositor and window events.
 164   if (desktop_window_) {
 165     aura::WindowTreeHost* window_host = desktop_window_->GetHost();
 166     // In the host destructor the compositor is destroyed before the window.
 167     if (window_host && window_host->compositor())
 168       window_host->compositor()->RemoveObserver(this);
 169     desktop_window_->RemoveObserver(this);
 170     desktop_window_ = NULL;
 171   }
 172
 173   // Stop timer.
 174   timer_.Stop();
 175
 176   callback.Run();
 177 }
 178
 179 void AuraWindowCaptureMachine::SetWindow(aura::Window* window) {
 180   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 181
 182   DCHECK(!desktop_window_);
 183   desktop_window_ = window;
 184
 185   // Start observing window events.
 186   desktop_window_->AddObserver(this);
 187
 188   // We must store this for the UMA reporting in DidCopyOutput() as
 189   // desktop_window_ might be destroyed at that point.
 190   screen_capture_ = window->IsRootWindow();
 191   IncrementDesktopCaptureCounter(screen_capture_ ? SCREEN_CAPTURER_CREATED
 192                                                  : WINDOW_CAPTURER_CREATED);
 193 }
 194
 195 void AuraWindowCaptureMachine::UpdateCaptureSize() {
 196   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 197   if (oracle_proxy_.get() && desktop_window_) {
 198      ui::Layer* layer = desktop_window_->layer();
 199      oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel(
 200          layer, layer->bounds().size()));
 201   }
 202   ClearCursorState();
 203 }
 204
 205 void AuraWindowCaptureMachine::Capture(bool dirty) {
 206   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 207
 208   // Do not capture if the desktop window is already destroyed.
 209   if (!desktop_window_)
 210     return;
 211
 212   scoped_refptr<media::VideoFrame> frame;
 213   ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
 214
 215   const base::TimeTicks start_time = base::TimeTicks::Now();
 216   const VideoCaptureOracle::Event event =
 217       dirty ? VideoCaptureOracle::kCompositorUpdate
 218             : VideoCaptureOracle::kTimerPoll;
 219   if (oracle_proxy_->ObserveEventAndDecideCapture(
 220           event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) {
 221     scoped_ptr<cc::CopyOutputRequest> request =
 222         cc::CopyOutputRequest::CreateRequest(
 223             base::Bind(&AuraWindowCaptureMachine::DidCopyOutput,
 224                        AsWeakPtr(), frame, start_time, capture_frame_cb));
 225     gfx::Rect window_rect = gfx::Rect(desktop_window_->bounds().width(),
 226                                       desktop_window_->bounds().height());
 227     request->set_area(window_rect);
 228     desktop_window_->layer()->RequestCopyOfOutput(request.Pass());
 229   }
 230 }
 231
 232 void AuraWindowCaptureMachine::DidCopyOutput(
 233     scoped_refptr<media::VideoFrame> video_frame,
 234     base::TimeTicks start_time,
 235     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 236     scoped_ptr<cc::CopyOutputResult> result) {
 237   static bool first_call = true;
 238
 239   bool succeeded = ProcessCopyOutputResponse(
 240       video_frame, start_time, capture_frame_cb, result.Pass());
 241
 242   base::TimeDelta capture_time = base::TimeTicks::Now() - start_time;
 243
 244   // The two UMA_ blocks must be put in its own scope since it creates a static
 245   // variable which expected constant histogram name.
 246   if (screen_capture_) {
 247     UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
 248   } else {
 249     UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
 250   }
 251
 252   if (first_call) {
 253     first_call = false;
 254     if (screen_capture_) {
 255       IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 256                                                : FIRST_SCREEN_CAPTURE_FAILED);
 257     } else {
 258       IncrementDesktopCaptureCounter(succeeded
 259                                          ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 260                                          : FIRST_WINDOW_CAPTURE_FAILED);
 261     }
 262   }
 263 }
 264
 265 bool AuraWindowCaptureMachine::ProcessCopyOutputResponse(
 266     scoped_refptr<media::VideoFrame> video_frame,
 267     base::TimeTicks start_time,
 268     const ThreadSafeCaptureOracle::CaptureFrameCallback& capture_frame_cb,
 269     scoped_ptr<cc::CopyOutputResult> result) {
 270   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 271
 272   if (result->IsEmpty() || result->size().IsEmpty() || !desktop_window_)
 273     return false;
 274
 275   if (capture_params_.requested_format.pixel_format ==
 276       media::PIXEL_FORMAT_TEXTURE) {
 277     DCHECK(!video_frame.get());
 278     cc::TextureMailbox texture_mailbox;
 279     scoped_ptr<cc::SingleReleaseCallback> release_callback;
 280     result->TakeTexture(&texture_mailbox, &release_callback);
 281     DCHECK(texture_mailbox.IsTexture());
 282     if (!texture_mailbox.IsTexture())
 283       return false;
 284     video_frame = media::VideoFrame::WrapNativeTexture(
 285         make_scoped_ptr(new gpu::MailboxHolder(texture_mailbox.mailbox(),
 286                                                texture_mailbox.target(),
 287                                                texture_mailbox.sync_point())),
 288         base::Bind(&RunSingleReleaseCallback, base::Passed(&release_callback)),
 289         result->size(), gfx::Rect(result->size()), result->size(),
 290         base::TimeDelta(), false);
 291     capture_frame_cb.Run(video_frame, start_time, true);
 292     return true;
 293   } else {
 294     DCHECK(video_frame.get());
 295   }
 296
 297   // Compute the dest size we want after the letterboxing resize. Make the
 298   // coordinates and sizes even because we letterbox in YUV space
 299   // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
 300   // line up correctly.
 301   // The video frame's visible_rect() and the result's size() are both physical
 302   // pixels.
 303   gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
 304       video_frame->visible_rect(), result->size());
 305   region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
 306                               region_in_frame.y() & ~1,
 307                               region_in_frame.width() & ~1,
 308                               region_in_frame.height() & ~1);
 309   if (region_in_frame.IsEmpty())
 310     return false;
 311
 312   ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
 313   GLHelper* gl_helper = factory->GetGLHelper();
 314   if (!gl_helper)
 315     return false;
 316
 317   cc::TextureMailbox texture_mailbox;
 318   scoped_ptr<cc::SingleReleaseCallback> release_callback;
 319   result->TakeTexture(&texture_mailbox, &release_callback);
 320   DCHECK(texture_mailbox.IsTexture());
 321   if (!texture_mailbox.IsTexture())
 322     return false;
 323
 324   gfx::Rect result_rect(result->size());
 325   if (!yuv_readback_pipeline_ ||
 326       yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
 327       yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
 328       yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
 329     yuv_readback_pipeline_.reset(
 330         gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
 331                                              result_rect.size(),
 332                                              result_rect,
 333                                              region_in_frame.size(),
 334                                              true,
 335                                              true));
 336   }
 337
 338   gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
 339   yuv_readback_pipeline_->ReadbackYUV(
 340       texture_mailbox.mailbox(),
 341       texture_mailbox.sync_point(),
 342       video_frame.get(),
 343       region_in_frame.origin(),
 344       base::Bind(&CopyOutputFinishedForVideo,
 345                  start_time,
 346                  capture_frame_cb,
 347                  video_frame,
 348                  scaled_cursor_bitmap_,
 349                  cursor_position_in_frame,
 350                  base::Passed(&release_callback)));
 351   return true;
 352 }
 353
 354 gfx::Point AuraWindowCaptureMachine::UpdateCursorState(
 355     const gfx::Rect& region_in_frame) {
 356   const gfx::Rect desktop_bounds = desktop_window_->layer()->bounds();
 357   if (desktop_bounds.IsEmpty()) {
 358     // Return early to prevent divide-by-zero in calculations below.
 359     ClearCursorState();
 360     return gfx::Point();
 361   }
 362
 363   gfx::NativeCursor cursor =
 364       desktop_window_->GetHost()->last_cursor();
 365   if (last_cursor_ != cursor ||
 366       desktop_size_when_cursor_last_updated_ != desktop_bounds.size()) {
 367     SkBitmap cursor_bitmap;
 368     if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
 369       const int scaled_width = cursor_bitmap.width() *
 370           region_in_frame.width() / desktop_bounds.width();
 371       const int scaled_height = cursor_bitmap.height() *
 372           region_in_frame.height() / desktop_bounds.height();
 373       if (scaled_width <= 0 || scaled_height <= 0) {
 374         ClearCursorState();
 375         return gfx::Point();
 376       }
 377       scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
 378           cursor_bitmap,
 379           skia::ImageOperations::RESIZE_BEST,
 380           scaled_width,
 381           scaled_height);
 382       last_cursor_ = cursor;
 383       desktop_size_when_cursor_last_updated_ = desktop_bounds.size();
 384     } else {
 385       // Clear cursor state if ui::GetCursorBitmap failed so that we do not
 386       // render cursor on the captured frame.
 387       ClearCursorState();
 388     }
 389   }
 390
 391   gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
 392   aura::client::GetScreenPositionClient(desktop_window_->GetRootWindow())->
 393       ConvertPointFromScreen(desktop_window_, &cursor_position);
 394   const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
 395       desktop_window_->layer(), cursor_hot_point_);
 396   cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
 397                          -desktop_bounds.y() - hot_point_in_dip.y());
 398   return gfx::Point(
 399       region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
 400           desktop_bounds.width(),
 401       region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
 402           desktop_bounds.height());
 403 }
 404
 405 void AuraWindowCaptureMachine::ClearCursorState() {
 406   last_cursor_ = ui::Cursor();
 407   desktop_size_when_cursor_last_updated_ = gfx::Size();
 408   cursor_hot_point_ = gfx::Point();
 409   scaled_cursor_bitmap_.reset();
 410 }
 411
 412 void AuraWindowCaptureMachine::OnWindowBoundsChanged(
 413     aura::Window* window,
 414     const gfx::Rect& old_bounds,
 415     const gfx::Rect& new_bounds) {
 416   DCHECK(desktop_window_ && window == desktop_window_);
 417
 418   // Post task to update capture size on UI thread.
 419   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 420       &AuraWindowCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
 421 }
 422
 423 void AuraWindowCaptureMachine::OnWindowDestroyed(aura::Window* window) {
 424   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 425
 426   Stop(base::Bind(&base::DoNothing));
 427
 428   oracle_proxy_->ReportError("OnWindowDestroyed()");
 429 }
 430
 431 void AuraWindowCaptureMachine::OnWindowAddedToRootWindow(
 432     aura::Window* window) {
 433   DCHECK(window == desktop_window_);
 434   window->GetHost()->compositor()->AddObserver(this);
 435 }
 436
 437 void AuraWindowCaptureMachine::OnWindowRemovingFromRootWindow(
 438     aura::Window* window,
 439     aura::Window* new_root) {
 440   DCHECK(window == desktop_window_);
 441   window->GetHost()->compositor()->RemoveObserver(this);
 442 }
 443
 444 void AuraWindowCaptureMachine::OnCompositingEnded(
 445     ui::Compositor* compositor) {
 446   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 447       &AuraWindowCaptureMachine::Capture, AsWeakPtr(), true));
 448 }
 449
 450 }  // namespace content