content/browser/media/capture/aura_window_capture_machine.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/aura_window_capture_machine.h"
   6
   7 #include "base/logging.h"
   8 #include "base/metrics/histogram.h"
   9 #include "base/timer/timer.h"
  10 #include "cc/output/copy_output_request.h"
  11 #include "cc/output/copy_output_result.h"
  12 #include "content/browser/compositor/image_transport_factory.h"
  13 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  14 #include "content/common/gpu/client/gl_helper.h"
  15 #include "content/public/browser/browser_thread.h"
  16 #include "content/public/browser/power_save_blocker.h"
  17 #include "media/base/video_capture_types.h"
  18 #include "media/base/video_util.h"
  19 #include "media/capture/content/thread_safe_capture_oracle.h"
  20 #include "media/capture/content/video_capture_oracle.h"
  21 #include "skia/ext/image_operations.h"
  22 #include "third_party/skia/include/core/SkBitmap.h"
  23 #include "ui/aura/client/screen_position_client.h"
  24 #include "ui/aura/env.h"
  25 #include "ui/aura/window.h"
  26 #include "ui/aura/window_observer.h"
  27 #include "ui/aura/window_tree_host.h"
  28 #include "ui/base/cursor/cursors_aura.h"
  29 #include "ui/compositor/compositor.h"
  30 #include "ui/compositor/dip_util.h"
  31 #include "ui/compositor/layer.h"
  32 #include "ui/gfx/screen.h"
  33
  34 namespace content {
  35
  36 namespace {
  37
  38 int clip_byte(int x) {
  39   return std::max(0, std::min(x, 255));
  40 }
  41
  42 int alpha_blend(int alpha, int src, int dst) {
  43   return (src * alpha + dst * (255 - alpha)) / 255;
  44 }
  45
  46 // Helper function to composite a cursor bitmap on a YUV420 video frame.
  47 void RenderCursorOnVideoFrame(
  48     const scoped_refptr<media::VideoFrame>& target,
  49     const SkBitmap& cursor_bitmap,
  50     const gfx::Point& cursor_position) {
  51   DCHECK(target.get());
  52   DCHECK(!cursor_bitmap.isNull());
  53
  54   gfx::Rect rect = gfx::IntersectRects(
  55       gfx::Rect(cursor_bitmap.width(), cursor_bitmap.height()) +
  56           gfx::Vector2d(cursor_position.x(), cursor_position.y()),
  57       target->visible_rect());
  58
  59   cursor_bitmap.lockPixels();
  60   for (int y = rect.y(); y < rect.bottom(); ++y) {
  61     int cursor_y = y - cursor_position.y();
  62     uint8* yplane = target->data(media::VideoFrame::kYPlane) +
  63         y * target->row_bytes(media::VideoFrame::kYPlane);
  64     uint8* uplane = target->data(media::VideoFrame::kUPlane) +
  65         (y / 2) * target->row_bytes(media::VideoFrame::kUPlane);
  66     uint8* vplane = target->data(media::VideoFrame::kVPlane) +
  67         (y / 2) * target->row_bytes(media::VideoFrame::kVPlane);
  68     for (int x = rect.x(); x < rect.right(); ++x) {
  69       int cursor_x = x - cursor_position.x();
  70       SkColor color = cursor_bitmap.getColor(cursor_x, cursor_y);
  71       int alpha = SkColorGetA(color);
  72       int color_r = SkColorGetR(color);
  73       int color_g = SkColorGetG(color);
  74       int color_b = SkColorGetB(color);
  75       int color_y = clip_byte(((color_r * 66 + color_g * 129 + color_b * 25 +
  76                                 128) >> 8) + 16);
  77       yplane[x] = alpha_blend(alpha, color_y, yplane[x]);
  78
  79       // Only sample U and V at even coordinates.
  80       if ((x % 2 == 0) && (y % 2 == 0)) {
  81         int color_u = clip_byte(((color_r * -38 + color_g * -74 +
  82                                   color_b * 112 + 128) >> 8) + 128);
  83         int color_v = clip_byte(((color_r * 112 + color_g * -94 +
  84                                   color_b * -18 + 128) >> 8) + 128);
  85         uplane[x / 2] = alpha_blend(alpha, color_u, uplane[x / 2]);
  86         vplane[x / 2] = alpha_blend(alpha, color_v, vplane[x / 2]);
  87       }
  88     }
  89   }
  90   cursor_bitmap.unlockPixels();
  91 }
  92
  93 using CaptureFrameCallback =
  94     media::ThreadSafeCaptureOracle::CaptureFrameCallback;
  95
  96 void CopyOutputFinishedForVideo(
  97     base::TimeTicks start_time,
  98     const CaptureFrameCallback& capture_frame_cb,
  99     const scoped_refptr<media::VideoFrame>& target,
 100     const SkBitmap& cursor_bitmap,
 101     const gfx::Point& cursor_position,
 102     scoped_ptr<cc::SingleReleaseCallback> release_callback,
 103     bool result) {
 104   if (!cursor_bitmap.isNull())
 105     RenderCursorOnVideoFrame(target, cursor_bitmap, cursor_position);
 106   release_callback->Run(0, false);
 107   capture_frame_cb.Run(target, start_time, result);
 108 }
 109
 110 void RunSingleReleaseCallback(scoped_ptr<cc::SingleReleaseCallback> cb,
 111                               uint32 sync_point) {
 112   cb->Run(sync_point, false);
 113 }
 114
 115 }  // namespace
 116
 117 AuraWindowCaptureMachine::AuraWindowCaptureMachine()
 118     : desktop_window_(NULL),
 119       timer_(true, true),
 120       screen_capture_(false) {}
 121
 122 AuraWindowCaptureMachine::~AuraWindowCaptureMachine() {}
 123
 124 void AuraWindowCaptureMachine::Start(
 125   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 126   const media::VideoCaptureParams& params,
 127   const base::Callback<void(bool)> callback) {
 128   // Starts the capture machine asynchronously.
 129   BrowserThread::PostTaskAndReplyWithResult(
 130       BrowserThread::UI,
 131       FROM_HERE,
 132       base::Bind(&AuraWindowCaptureMachine::InternalStart,
 133                  base::Unretained(this),
 134                  oracle_proxy,
 135                  params),
 136       callback);
 137 }
 138
 139 bool AuraWindowCaptureMachine::InternalStart(
 140     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
 141     const media::VideoCaptureParams& params) {
 142   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 143
 144   // The window might be destroyed between SetWindow() and Start().
 145   if (!desktop_window_)
 146     return false;
 147
 148   // If the associated layer is already destroyed then return failure.
 149   ui::Layer* layer = desktop_window_->layer();
 150   if (!layer)
 151     return false;
 152
 153   DCHECK(oracle_proxy.get());
 154   oracle_proxy_ = oracle_proxy;
 155   capture_params_ = params;
 156
 157   // Update capture size.
 158   UpdateCaptureSize();
 159
 160   // Start observing compositor updates.
 161   if (desktop_window_->GetHost())
 162     desktop_window_->GetHost()->compositor()->AddObserver(this);
 163
 164   power_save_blocker_.reset(
 165       PowerSaveBlocker::Create(
 166           PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
 167           PowerSaveBlocker::kReasonOther,
 168           "DesktopCaptureDevice is running").release());
 169
 170   // Starts timer.
 171   timer_.Start(FROM_HERE,
 172                std::max(oracle_proxy_->min_capture_period(),
 173                         base::TimeDelta::FromMilliseconds(media::
 174                             VideoCaptureOracle::kMinTimerPollPeriodMillis)),
 175                base::Bind(&AuraWindowCaptureMachine::Capture, AsWeakPtr(),
 176                           false));
 177
 178   return true;
 179 }
 180
 181 void AuraWindowCaptureMachine::Stop(const base::Closure& callback) {
 182   // Stops the capture machine asynchronously.
 183   BrowserThread::PostTask(
 184       BrowserThread::UI, FROM_HERE, base::Bind(
 185           &AuraWindowCaptureMachine::InternalStop,
 186           base::Unretained(this),
 187           callback));
 188 }
 189
 190 void AuraWindowCaptureMachine::InternalStop(const base::Closure& callback) {
 191   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 192   power_save_blocker_.reset();
 193
 194   // Stop observing compositor and window events.
 195   if (desktop_window_) {
 196     aura::WindowTreeHost* window_host = desktop_window_->GetHost();
 197     // In the host destructor the compositor is destroyed before the window.
 198     if (window_host && window_host->compositor())
 199       window_host->compositor()->RemoveObserver(this);
 200     desktop_window_->RemoveObserver(this);
 201     desktop_window_ = NULL;
 202   }
 203
 204   // Stop timer.
 205   timer_.Stop();
 206
 207   callback.Run();
 208 }
 209
 210 void AuraWindowCaptureMachine::SetWindow(aura::Window* window) {
 211   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 212
 213   DCHECK(!desktop_window_);
 214   desktop_window_ = window;
 215
 216   // Start observing window events.
 217   desktop_window_->AddObserver(this);
 218
 219   // We must store this for the UMA reporting in DidCopyOutput() as
 220   // desktop_window_ might be destroyed at that point.
 221   screen_capture_ = window->IsRootWindow();
 222   IncrementDesktopCaptureCounter(screen_capture_ ? SCREEN_CAPTURER_CREATED
 223                                                  : WINDOW_CAPTURER_CREATED);
 224 }
 225
 226 void AuraWindowCaptureMachine::UpdateCaptureSize() {
 227   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 228   if (oracle_proxy_.get() && desktop_window_) {
 229      ui::Layer* layer = desktop_window_->layer();
 230      oracle_proxy_->UpdateCaptureSize(ui::ConvertSizeToPixel(
 231          layer, layer->bounds().size()));
 232   }
 233   ClearCursorState();
 234 }
 235
 236 void AuraWindowCaptureMachine::Capture(bool dirty) {
 237   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 238
 239   // Do not capture if the desktop window is already destroyed.
 240   if (!desktop_window_)
 241     return;
 242
 243   scoped_refptr<media::VideoFrame> frame;
 244   media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
 245
 246   // TODO(miu): Need to fix this so the compositor is providing the presentation
 247   // timestamps and damage regions, to leverage the frame timestamp rewriting
 248   // logic.  http://crbug.com/492839
 249   const base::TimeTicks start_time = base::TimeTicks::Now();
 250   const media::VideoCaptureOracle::Event event =
 251       dirty ? media::VideoCaptureOracle::kCompositorUpdate
 252             : media::VideoCaptureOracle::kTimerPoll;
 253   if (oracle_proxy_->ObserveEventAndDecideCapture(
 254           event, gfx::Rect(), start_time, &frame, &capture_frame_cb)) {
 255     scoped_ptr<cc::CopyOutputRequest> request =
 256         cc::CopyOutputRequest::CreateRequest(
 257             base::Bind(&AuraWindowCaptureMachine::DidCopyOutput,
 258                        AsWeakPtr(), frame, start_time, capture_frame_cb));
 259     gfx::Rect window_rect = gfx::Rect(desktop_window_->bounds().width(),
 260                                       desktop_window_->bounds().height());
 261     request->set_area(window_rect);
 262     desktop_window_->layer()->RequestCopyOfOutput(request.Pass());
 263   }
 264 }
 265
 266 void AuraWindowCaptureMachine::DidCopyOutput(
 267     scoped_refptr<media::VideoFrame> video_frame,
 268     base::TimeTicks start_time,
 269     const CaptureFrameCallback& capture_frame_cb,
 270     scoped_ptr<cc::CopyOutputResult> result) {
 271   static bool first_call = true;
 272
 273   bool succeeded = ProcessCopyOutputResponse(
 274       video_frame, start_time, capture_frame_cb, result.Pass());
 275
 276   base::TimeDelta capture_time = base::TimeTicks::Now() - start_time;
 277
 278   // The two UMA_ blocks must be put in its own scope since it creates a static
 279   // variable which expected constant histogram name.
 280   if (screen_capture_) {
 281     UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
 282   } else {
 283     UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
 284   }
 285
 286   if (first_call) {
 287     first_call = false;
 288     if (screen_capture_) {
 289       IncrementDesktopCaptureCounter(succeeded ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 290                                                : FIRST_SCREEN_CAPTURE_FAILED);
 291     } else {
 292       IncrementDesktopCaptureCounter(succeeded
 293                                          ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 294                                          : FIRST_WINDOW_CAPTURE_FAILED);
 295     }
 296   }
 297 }
 298
 299 bool AuraWindowCaptureMachine::ProcessCopyOutputResponse(
 300     scoped_refptr<media::VideoFrame> video_frame,
 301     base::TimeTicks start_time,
 302     const CaptureFrameCallback& capture_frame_cb,
 303     scoped_ptr<cc::CopyOutputResult> result) {
 304   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 305
 306   if (result->IsEmpty() || result->size().IsEmpty() || !desktop_window_)
 307     return false;
 308
 309   if (capture_params_.requested_format.pixel_storage ==
 310       media::PIXEL_STORAGE_TEXTURE) {
 311     DCHECK_EQ(media::VIDEO_CAPTURE_PIXEL_FORMAT_ARGB,
 312               capture_params_.requested_format.pixel_format);
 313     DCHECK(!video_frame.get());
 314     cc::TextureMailbox texture_mailbox;
 315     scoped_ptr<cc::SingleReleaseCallback> release_callback;
 316     result->TakeTexture(&texture_mailbox, &release_callback);
 317     DCHECK(texture_mailbox.IsTexture());
 318     if (!texture_mailbox.IsTexture())
 319       return false;
 320     video_frame = media::VideoFrame::WrapNativeTexture(
 321         media::PIXEL_FORMAT_ARGB,
 322         gpu::MailboxHolder(texture_mailbox.mailbox(), texture_mailbox.target(),
 323                            texture_mailbox.sync_point()),
 324         base::Bind(&RunSingleReleaseCallback, base::Passed(&release_callback)),
 325         result->size(), gfx::Rect(result->size()), result->size(),
 326         base::TimeDelta());
 327     capture_frame_cb.Run(video_frame, start_time, true);
 328     return true;
 329   } else {
 330     DCHECK(video_frame.get());
 331   }
 332
 333   // Compute the dest size we want after the letterboxing resize. Make the
 334   // coordinates and sizes even because we letterbox in YUV space
 335   // (see CopyRGBToVideoFrame). They need to be even for the UV samples to
 336   // line up correctly.
 337   // The video frame's visible_rect() and the result's size() are both physical
 338   // pixels.
 339   gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
 340       video_frame->visible_rect(), result->size());
 341   region_in_frame = gfx::Rect(region_in_frame.x() & ~1,
 342                               region_in_frame.y() & ~1,
 343                               region_in_frame.width() & ~1,
 344                               region_in_frame.height() & ~1);
 345   if (region_in_frame.IsEmpty())
 346     return false;
 347
 348   ImageTransportFactory* factory = ImageTransportFactory::GetInstance();
 349   GLHelper* gl_helper = factory->GetGLHelper();
 350   if (!gl_helper)
 351     return false;
 352
 353   cc::TextureMailbox texture_mailbox;
 354   scoped_ptr<cc::SingleReleaseCallback> release_callback;
 355   result->TakeTexture(&texture_mailbox, &release_callback);
 356   DCHECK(texture_mailbox.IsTexture());
 357   if (!texture_mailbox.IsTexture())
 358     return false;
 359
 360   gfx::Rect result_rect(result->size());
 361   if (!yuv_readback_pipeline_ ||
 362       yuv_readback_pipeline_->scaler()->SrcSize() != result_rect.size() ||
 363       yuv_readback_pipeline_->scaler()->SrcSubrect() != result_rect ||
 364       yuv_readback_pipeline_->scaler()->DstSize() != region_in_frame.size()) {
 365     yuv_readback_pipeline_.reset(
 366         gl_helper->CreateReadbackPipelineYUV(GLHelper::SCALER_QUALITY_FAST,
 367                                              result_rect.size(),
 368                                              result_rect,
 369                                              region_in_frame.size(),
 370                                              true,
 371                                              true));
 372   }
 373
 374   gfx::Point cursor_position_in_frame = UpdateCursorState(region_in_frame);
 375   yuv_readback_pipeline_->ReadbackYUV(
 376       texture_mailbox.mailbox(),
 377       texture_mailbox.sync_point(),
 378       video_frame.get(),
 379       region_in_frame.origin(),
 380       base::Bind(&CopyOutputFinishedForVideo,
 381                  start_time,
 382                  capture_frame_cb,
 383                  video_frame,
 384                  scaled_cursor_bitmap_,
 385                  cursor_position_in_frame,
 386                  base::Passed(&release_callback)));
 387   return true;
 388 }
 389
 390 gfx::Point AuraWindowCaptureMachine::UpdateCursorState(
 391     const gfx::Rect& region_in_frame) {
 392   const gfx::Rect desktop_bounds = desktop_window_->layer()->bounds();
 393   if (desktop_bounds.IsEmpty()) {
 394     // Return early to prevent divide-by-zero in calculations below.
 395     ClearCursorState();
 396     return gfx::Point();
 397   }
 398
 399   gfx::NativeCursor cursor =
 400       desktop_window_->GetHost()->last_cursor();
 401   if (last_cursor_ != cursor ||
 402       desktop_size_when_cursor_last_updated_ != desktop_bounds.size()) {
 403     SkBitmap cursor_bitmap;
 404     if (ui::GetCursorBitmap(cursor, &cursor_bitmap, &cursor_hot_point_)) {
 405       const int scaled_width = cursor_bitmap.width() *
 406           region_in_frame.width() / desktop_bounds.width();
 407       const int scaled_height = cursor_bitmap.height() *
 408           region_in_frame.height() / desktop_bounds.height();
 409       if (scaled_width <= 0 || scaled_height <= 0) {
 410         ClearCursorState();
 411         return gfx::Point();
 412       }
 413       scaled_cursor_bitmap_ = skia::ImageOperations::Resize(
 414           cursor_bitmap,
 415           skia::ImageOperations::RESIZE_BEST,
 416           scaled_width,
 417           scaled_height);
 418       last_cursor_ = cursor;
 419       desktop_size_when_cursor_last_updated_ = desktop_bounds.size();
 420     } else {
 421       // Clear cursor state if ui::GetCursorBitmap failed so that we do not
 422       // render cursor on the captured frame.
 423       ClearCursorState();
 424     }
 425   }
 426
 427   gfx::Point cursor_position = aura::Env::GetInstance()->last_mouse_location();
 428   aura::client::GetScreenPositionClient(desktop_window_->GetRootWindow())->
 429       ConvertPointFromScreen(desktop_window_, &cursor_position);
 430   const gfx::Point hot_point_in_dip = ui::ConvertPointToDIP(
 431       desktop_window_->layer(), cursor_hot_point_);
 432   cursor_position.Offset(-desktop_bounds.x() - hot_point_in_dip.x(),
 433                          -desktop_bounds.y() - hot_point_in_dip.y());
 434   return gfx::Point(
 435       region_in_frame.x() + cursor_position.x() * region_in_frame.width() /
 436           desktop_bounds.width(),
 437       region_in_frame.y() + cursor_position.y() * region_in_frame.height() /
 438           desktop_bounds.height());
 439 }
 440
 441 void AuraWindowCaptureMachine::ClearCursorState() {
 442   last_cursor_ = ui::Cursor();
 443   desktop_size_when_cursor_last_updated_ = gfx::Size();
 444   cursor_hot_point_ = gfx::Point();
 445   scaled_cursor_bitmap_.reset();
 446 }
 447
 448 void AuraWindowCaptureMachine::OnWindowBoundsChanged(
 449     aura::Window* window,
 450     const gfx::Rect& old_bounds,
 451     const gfx::Rect& new_bounds) {
 452   DCHECK(desktop_window_ && window == desktop_window_);
 453
 454   // Post task to update capture size on UI thread.
 455   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 456       &AuraWindowCaptureMachine::UpdateCaptureSize, AsWeakPtr()));
 457 }
 458
 459 void AuraWindowCaptureMachine::OnWindowDestroyed(aura::Window* window) {
 460   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 461
 462   Stop(base::Bind(&base::DoNothing));
 463
 464   oracle_proxy_->ReportError("OnWindowDestroyed()");
 465 }
 466
 467 void AuraWindowCaptureMachine::OnWindowAddedToRootWindow(
 468     aura::Window* window) {
 469   DCHECK(window == desktop_window_);
 470   window->GetHost()->compositor()->AddObserver(this);
 471 }
 472
 473 void AuraWindowCaptureMachine::OnWindowRemovingFromRootWindow(
 474     aura::Window* window,
 475     aura::Window* new_root) {
 476   DCHECK(window == desktop_window_);
 477   window->GetHost()->compositor()->RemoveObserver(this);
 478 }
 479
 480 void AuraWindowCaptureMachine::OnCompositingEnded(
 481     ui::Compositor* compositor) {
 482   // TODO(miu): The CopyOutputRequest should be made earlier, at WillCommit().
 483   // http://crbug.com/492839
 484   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, base::Bind(
 485       &AuraWindowCaptureMachine::Capture, AsWeakPtr(), true));
 486 }
 487
 488 }  // namespace content