content/browser/media/capture/desktop_capture_device.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/desktop_capture_device.h"
   6
   7 #include "base/bind.h"
   8 #include "base/location.h"
   9 #include "base/logging.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/strings/string_number_conversions.h"
  12 #include "base/synchronization/lock.h"
  13 #include "base/threading/thread.h"
  14 #include "base/timer/timer.h"
  15 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  16 #include "content/public/browser/browser_thread.h"
  17 #include "content/public/browser/desktop_media_id.h"
  18 #include "content/public/browser/power_save_blocker.h"
  19 #include "media/base/video_util.h"
  20 #include "media/capture/content/capture_resolution_chooser.h"
  21 #include "third_party/libyuv/include/libyuv/scale_argb.h"
  22 #include "third_party/webrtc/modules/desktop_capture/cropping_window_capturer.h"
  23 #include "third_party/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h"
  24 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
  25 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
  26 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  27 #include "third_party/webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
  28 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
  29
  30 namespace content {
  31
  32 namespace {
  33
  34 // Maximum CPU time percentage of a single core that can be consumed for desktop
  35 // capturing. This means that on systems where screen scraping is slow we may
  36 // need to capture at frame rate lower than requested. This is necessary to keep
  37 // UI responsive.
  38 const int kMaximumCpuConsumptionPercentage = 50;
  39
  40 webrtc::DesktopRect ComputeLetterboxRect(
  41     const webrtc::DesktopSize& max_size,
  42     const webrtc::DesktopSize& source_size) {
  43   gfx::Rect result = media::ComputeLetterboxRegion(
  44       gfx::Rect(0, 0, max_size.width(), max_size.height()),
  45       gfx::Size(source_size.width(), source_size.height()));
  46   return webrtc::DesktopRect::MakeLTRB(
  47       result.x(), result.y(), result.right(), result.bottom());
  48 }
  49
  50 bool IsFrameUnpackedOrInverted(webrtc::DesktopFrame* frame) {
  51   return frame->stride() !=
  52       frame->size().width() * webrtc::DesktopFrame::kBytesPerPixel;
  53 }
  54
  55 }  // namespace
  56
  57 class DesktopCaptureDevice::Core : public webrtc::DesktopCapturer::Callback {
  58  public:
  59   Core(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
  60        scoped_ptr<webrtc::DesktopCapturer> capturer,
  61        DesktopMediaID::Type type);
  62   ~Core() override;
  63
  64   // Implementation of VideoCaptureDevice methods.
  65   void AllocateAndStart(const media::VideoCaptureParams& params,
  66                         scoped_ptr<Client> client);
  67
  68   void SetNotificationWindowId(gfx::NativeViewId window_id);
  69
  70  private:
  71
  72   // webrtc::DesktopCapturer::Callback interface
  73   webrtc::SharedMemory* CreateSharedMemory(size_t size) override;
  74   void OnCaptureCompleted(webrtc::DesktopFrame* frame) override;
  75
  76   // Method that is scheduled on |task_runner_| to be called on regular interval
  77   // to capture a frame.
  78   void OnCaptureTimer();
  79
  80   // Captures a frame and schedules timer for the next one.
  81   void CaptureFrameAndScheduleNext();
  82
  83   // Captures a single frame.
  84   void DoCapture();
  85
  86   // Task runner used for capturing operations.
  87   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  88
  89   // The underlying DesktopCapturer instance used to capture frames.
  90   scoped_ptr<webrtc::DesktopCapturer> desktop_capturer_;
  91
  92   // The device client which proxies device events to the controller. Accessed
  93   // on the task_runner_ thread.
  94   scoped_ptr<Client> client_;
  95
  96   // Requested video capture frame rate.
  97   float requested_frame_rate_;
  98
  99   // Size of frame most recently captured from the source.
 100   webrtc::DesktopSize previous_frame_size_;
 101
 102   // Determines the size of frames to deliver to the |client_|.
 103   scoped_ptr<media::CaptureResolutionChooser> resolution_chooser_;
 104
 105   // DesktopFrame into which captured frames are down-scaled and/or letterboxed,
 106   // depending upon the caller's requested capture capabilities. If frames can
 107   // be returned to the caller directly then this is NULL.
 108   scoped_ptr<webrtc::DesktopFrame> output_frame_;
 109
 110   // Timer used to capture the frame.
 111   base::OneShotTimer<Core> capture_timer_;
 112
 113   // True when waiting for |desktop_capturer_| to capture current frame.
 114   bool capture_in_progress_;
 115
 116   // True if the first capture call has returned. Used to log the first capture
 117   // result.
 118   bool first_capture_returned_;
 119
 120   // The type of the capturer.
 121   DesktopMediaID::Type capturer_type_;
 122
 123   scoped_ptr<webrtc::BasicDesktopFrame> black_frame_;
 124
 125   // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
 126   // screen from sleeping for the drive-by web.
 127   scoped_ptr<PowerSaveBlocker> power_save_blocker_;
 128
 129   DISALLOW_COPY_AND_ASSIGN(Core);
 130 };
 131
 132 DesktopCaptureDevice::Core::Core(
 133     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
 134     scoped_ptr<webrtc::DesktopCapturer> capturer,
 135     DesktopMediaID::Type type)
 136     : task_runner_(task_runner),
 137       desktop_capturer_(capturer.Pass()),
 138       capture_in_progress_(false),
 139       first_capture_returned_(false),
 140       capturer_type_(type) {
 141 }
 142
 143 DesktopCaptureDevice::Core::~Core() {
 144   DCHECK(task_runner_->BelongsToCurrentThread());
 145   client_.reset();
 146   output_frame_.reset();
 147   previous_frame_size_.set(0, 0);
 148   desktop_capturer_.reset();
 149 }
 150
 151 void DesktopCaptureDevice::Core::AllocateAndStart(
 152     const media::VideoCaptureParams& params,
 153     scoped_ptr<Client> client) {
 154   DCHECK(task_runner_->BelongsToCurrentThread());
 155   DCHECK_GT(params.requested_format.frame_size.GetArea(), 0);
 156   DCHECK_GT(params.requested_format.frame_rate, 0);
 157   DCHECK(desktop_capturer_);
 158   DCHECK(client.get());
 159   DCHECK(!client_.get());
 160
 161   client_ = client.Pass();
 162   requested_frame_rate_ = params.requested_format.frame_rate;
 163   resolution_chooser_.reset(new media::CaptureResolutionChooser(
 164       params.requested_format.frame_size,
 165       params.resolution_change_policy));
 166
 167   power_save_blocker_.reset(
 168       PowerSaveBlocker::Create(
 169           PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
 170           PowerSaveBlocker::kReasonOther,
 171           "DesktopCaptureDevice is running").release());
 172
 173   desktop_capturer_->Start(this);
 174
 175   CaptureFrameAndScheduleNext();
 176 }
 177
 178 void DesktopCaptureDevice::Core::SetNotificationWindowId(
 179     gfx::NativeViewId window_id) {
 180   DCHECK(task_runner_->BelongsToCurrentThread());
 181   DCHECK(window_id);
 182   desktop_capturer_->SetExcludedWindow(window_id);
 183 }
 184
 185 webrtc::SharedMemory*
 186 DesktopCaptureDevice::Core::CreateSharedMemory(size_t size) {
 187   return NULL;
 188 }
 189
 190 void DesktopCaptureDevice::Core::OnCaptureCompleted(
 191     webrtc::DesktopFrame* frame) {
 192   DCHECK(task_runner_->BelongsToCurrentThread());
 193   DCHECK(capture_in_progress_);
 194
 195   if (!first_capture_returned_) {
 196     first_capture_returned_ = true;
 197     if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 198       IncrementDesktopCaptureCounter(frame ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 199                                            : FIRST_SCREEN_CAPTURE_FAILED);
 200     } else {
 201       IncrementDesktopCaptureCounter(frame ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 202                                            : FIRST_WINDOW_CAPTURE_FAILED);
 203     }
 204   }
 205
 206   capture_in_progress_ = false;
 207
 208   if (!frame) {
 209     std::string log("Failed to capture a frame.");
 210     LOG(ERROR) << log;
 211     client_->OnError(log);
 212     return;
 213   }
 214
 215   if (!client_)
 216     return;
 217
 218   base::TimeDelta capture_time(
 219       base::TimeDelta::FromMilliseconds(frame->capture_time_ms()));
 220
 221   // The two UMA_ blocks must be put in its own scope since it creates a static
 222   // variable which expected constant histogram name.
 223   if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 224     UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
 225   } else {
 226     UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
 227   }
 228
 229   scoped_ptr<webrtc::DesktopFrame> owned_frame(frame);
 230
 231   // If the frame size has changed, drop the output frame (if any), and
 232   // determine the new output size.
 233   if (!previous_frame_size_.equals(frame->size())) {
 234     output_frame_.reset();
 235     resolution_chooser_->SetSourceSize(gfx::Size(frame->size().width(),
 236                                                  frame->size().height()));
 237     previous_frame_size_ = frame->size();
 238   }
 239   // Align to 2x2 pixel boundaries, as required by OnIncomingCapturedData() so
 240   // it can convert the frame to I420 format.
 241   const webrtc::DesktopSize output_size(
 242       resolution_chooser_->capture_size().width() & ~1,
 243       resolution_chooser_->capture_size().height() & ~1);
 244   if (output_size.is_empty())
 245     return;
 246
 247   // On OSX We receive a 1x1 frame when the shared window is minimized. It
 248   // cannot be subsampled to I420 and will be dropped downstream. So we replace
 249   // it with a black frame to avoid the video appearing frozen at the last
 250   // frame.
 251   if (frame->size().width() == 1 || frame->size().height() == 1) {
 252     if (!black_frame_.get()) {
 253       black_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 254       memset(black_frame_->data(),
 255              0,
 256              black_frame_->stride() * black_frame_->size().height());
 257     }
 258     owned_frame.reset();
 259     frame = black_frame_.get();
 260   }
 261
 262   size_t output_bytes = output_size.width() * output_size.height() *
 263       webrtc::DesktopFrame::kBytesPerPixel;
 264   const uint8_t* output_data = NULL;
 265
 266   if (!frame->size().equals(output_size)) {
 267     // Down-scale and/or letterbox to the target format if the frame does not
 268     // match the output size.
 269
 270     // Allocate a buffer of the correct size to scale the frame into.
 271     // |output_frame_| is cleared whenever the output size changes, so we don't
 272     // need to worry about clearing out stale pixel data in letterboxed areas.
 273     if (!output_frame_) {
 274       output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 275       memset(output_frame_->data(), 0, output_bytes);
 276     }
 277     DCHECK(output_frame_->size().equals(output_size));
 278
 279     // TODO(wez): Optimize this to scale only changed portions of the output,
 280     // using ARGBScaleClip().
 281     const webrtc::DesktopRect output_rect =
 282         ComputeLetterboxRect(output_size, frame->size());
 283     uint8_t* output_rect_data = output_frame_->data() +
 284         output_frame_->stride() * output_rect.top() +
 285         webrtc::DesktopFrame::kBytesPerPixel * output_rect.left();
 286     libyuv::ARGBScale(frame->data(), frame->stride(),
 287                       frame->size().width(), frame->size().height(),
 288                       output_rect_data, output_frame_->stride(),
 289                       output_rect.width(), output_rect.height(),
 290                       libyuv::kFilterBilinear);
 291     output_data = output_frame_->data();
 292   } else if (IsFrameUnpackedOrInverted(frame)) {
 293     // If |frame| is not packed top-to-bottom then create a packed top-to-bottom
 294     // copy.
 295     // This is required if the frame is inverted (see crbug.com/306876), or if
 296     // |frame| is cropped form a larger frame (see crbug.com/437740).
 297     if (!output_frame_) {
 298       output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 299       memset(output_frame_->data(), 0, output_bytes);
 300     }
 301
 302     output_frame_->CopyPixelsFrom(
 303         *frame,
 304         webrtc::DesktopVector(),
 305         webrtc::DesktopRect::MakeSize(frame->size()));
 306     output_data = output_frame_->data();
 307   } else {
 308     // If the captured frame matches the output size, we can return the pixel
 309     // data directly.
 310     output_data = frame->data();
 311   }
 312
 313   client_->OnIncomingCapturedData(
 314       output_data, output_bytes,
 315       media::VideoCaptureFormat(
 316           gfx::Size(output_size.width(), output_size.height()),
 317           requested_frame_rate_, media::VIDEO_CAPTURE_PIXEL_FORMAT_ARGB),
 318       0, base::TimeTicks::Now());
 319 }
 320
 321 void DesktopCaptureDevice::Core::OnCaptureTimer() {
 322   DCHECK(task_runner_->BelongsToCurrentThread());
 323
 324   if (!client_)
 325     return;
 326
 327   CaptureFrameAndScheduleNext();
 328 }
 329
 330 void DesktopCaptureDevice::Core::CaptureFrameAndScheduleNext() {
 331   DCHECK(task_runner_->BelongsToCurrentThread());
 332
 333   base::TimeTicks started_time = base::TimeTicks::Now();
 334   DoCapture();
 335   base::TimeDelta last_capture_duration = base::TimeTicks::Now() - started_time;
 336
 337   // Limit frame-rate to reduce CPU consumption.
 338   base::TimeDelta capture_period = std::max(
 339     (last_capture_duration * 100) / kMaximumCpuConsumptionPercentage,
 340     base::TimeDelta::FromMicroseconds(static_cast<int64>(
 341         1000000.0 / requested_frame_rate_ + 0.5 /* round to nearest int */)));
 342
 343   // Schedule a task for the next frame.
 344   capture_timer_.Start(FROM_HERE, capture_period - last_capture_duration,
 345                        this, &Core::OnCaptureTimer);
 346 }
 347
 348 void DesktopCaptureDevice::Core::DoCapture() {
 349   DCHECK(task_runner_->BelongsToCurrentThread());
 350   DCHECK(!capture_in_progress_);
 351
 352   capture_in_progress_ = true;
 353   desktop_capturer_->Capture(webrtc::DesktopRegion());
 354
 355   // Currently only synchronous implementations of DesktopCapturer are
 356   // supported.
 357   DCHECK(!capture_in_progress_);
 358 }
 359
 360 // static
 361 scoped_ptr<media::VideoCaptureDevice> DesktopCaptureDevice::Create(
 362     const DesktopMediaID& source) {
 363   webrtc::DesktopCaptureOptions options =
 364       webrtc::DesktopCaptureOptions::CreateDefault();
 365   // Leave desktop effects enabled during WebRTC captures.
 366   options.set_disable_effects(false);
 367
 368 #if defined(OS_WIN)
 369       options.set_allow_use_magnification_api(true);
 370 #endif
 371
 372   scoped_ptr<webrtc::DesktopCapturer> capturer;
 373
 374   switch (source.type) {
 375     case DesktopMediaID::TYPE_SCREEN: {
 376       scoped_ptr<webrtc::ScreenCapturer> screen_capturer(
 377           webrtc::ScreenCapturer::Create(options));
 378       if (screen_capturer && screen_capturer->SelectScreen(source.id)) {
 379         capturer.reset(new webrtc::DesktopAndCursorComposer(
 380             screen_capturer.release(),
 381             webrtc::MouseCursorMonitor::CreateForScreen(options, source.id)));
 382         IncrementDesktopCaptureCounter(SCREEN_CAPTURER_CREATED);
 383       }
 384       break;
 385     }
 386
 387     case DesktopMediaID::TYPE_WINDOW: {
 388       scoped_ptr<webrtc::WindowCapturer> window_capturer(
 389           webrtc::CroppingWindowCapturer::Create(options));
 390       if (window_capturer && window_capturer->SelectWindow(source.id)) {
 391         window_capturer->BringSelectedWindowToFront();
 392         capturer.reset(new webrtc::DesktopAndCursorComposer(
 393             window_capturer.release(),
 394             webrtc::MouseCursorMonitor::CreateForWindow(options, source.id)));
 395         IncrementDesktopCaptureCounter(WINDOW_CAPTURER_CREATED);
 396       }
 397       break;
 398     }
 399
 400     default: {
 401       NOTREACHED();
 402     }
 403   }
 404
 405   scoped_ptr<media::VideoCaptureDevice> result;
 406   if (capturer)
 407     result.reset(new DesktopCaptureDevice(capturer.Pass(), source.type));
 408
 409   return result.Pass();
 410 }
 411
 412 DesktopCaptureDevice::~DesktopCaptureDevice() {
 413   DCHECK(!core_);
 414 }
 415
 416 void DesktopCaptureDevice::AllocateAndStart(
 417     const media::VideoCaptureParams& params,
 418     scoped_ptr<Client> client) {
 419   thread_.task_runner()->PostTask(
 420       FROM_HERE,
 421       base::Bind(&Core::AllocateAndStart, base::Unretained(core_.get()), params,
 422                  base::Passed(&client)));
 423 }
 424
 425 void DesktopCaptureDevice::StopAndDeAllocate() {
 426   if (core_) {
 427     thread_.task_runner()->DeleteSoon(FROM_HERE, core_.release());
 428     thread_.Stop();
 429   }
 430 }
 431
 432 void DesktopCaptureDevice::SetNotificationWindowId(
 433     gfx::NativeViewId window_id) {
 434   // This may be called after the capturer has been stopped.
 435   if (!core_)
 436     return;
 437   thread_.task_runner()->PostTask(
 438       FROM_HERE,
 439       base::Bind(&Core::SetNotificationWindowId,
 440                  base::Unretained(core_.get()),
 441                  window_id));
 442 }
 443
 444 DesktopCaptureDevice::DesktopCaptureDevice(
 445     scoped_ptr<webrtc::DesktopCapturer> capturer,
 446     DesktopMediaID::Type type)
 447     : thread_("desktopCaptureThread") {
 448 #if defined(OS_WIN)
 449   // On Windows the thread must be a UI thread.
 450   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_UI;
 451 #else
 452   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_DEFAULT;
 453 #endif
 454
 455   thread_.StartWithOptions(base::Thread::Options(thread_type, 0));
 456
 457   core_.reset(new Core(thread_.task_runner(), capturer.Pass(), type));
 458 }
 459
 460 }  // namespace content