content/browser/media/capture/desktop_capture_device.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/desktop_capture_device.h"
   6
   7 #include "base/bind.h"
   8 #include "base/location.h"
   9 #include "base/logging.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/strings/string_number_conversions.h"
  12 #include "base/synchronization/lock.h"
  13 #include "base/threading/thread.h"
  14 #include "base/timer/timer.h"
  15 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  16 #include "content/public/browser/browser_thread.h"
  17 #include "content/public/browser/desktop_media_id.h"
  18 #include "content/public/browser/power_save_blocker.h"
  19 #include "media/base/video_util.h"
  20 #include "third_party/libyuv/include/libyuv/scale_argb.h"
  21 #include "third_party/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h"
  22 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
  23 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
  24 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  25 #include "third_party/webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
  26 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
  27 #include "third_party/webrtc/modules/desktop_capture/window_capturer.h"
  28
  29 namespace content {
  30
  31 namespace {
  32
  33 // Maximum CPU time percentage of a single core that can be consumed for desktop
  34 // capturing. This means that on systems where screen scraping is slow we may
  35 // need to capture at frame rate lower than requested. This is necessary to keep
  36 // UI responsive.
  37 const int kMaximumCpuConsumptionPercentage = 50;
  38
  39 webrtc::DesktopRect ComputeLetterboxRect(
  40     const webrtc::DesktopSize& max_size,
  41     const webrtc::DesktopSize& source_size) {
  42   gfx::Rect result = media::ComputeLetterboxRegion(
  43       gfx::Rect(0, 0, max_size.width(), max_size.height()),
  44       gfx::Size(source_size.width(), source_size.height()));
  45   return webrtc::DesktopRect::MakeLTRB(
  46       result.x(), result.y(), result.right(), result.bottom());
  47 }
  48
  49 }  // namespace
  50
  51 class DesktopCaptureDevice::Core : public webrtc::DesktopCapturer::Callback {
  52  public:
  53   Core(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
  54        scoped_ptr<webrtc::DesktopCapturer> capturer,
  55        DesktopMediaID::Type type);
  56   virtual ~Core();
  57
  58   // Implementation of VideoCaptureDevice methods.
  59   void AllocateAndStart(const media::VideoCaptureParams& params,
  60                         scoped_ptr<Client> client);
  61
  62   void SetNotificationWindowId(gfx::NativeViewId window_id);
  63
  64  private:
  65
  66   // webrtc::DesktopCapturer::Callback interface
  67   virtual webrtc::SharedMemory* CreateSharedMemory(size_t size) OVERRIDE;
  68   virtual void OnCaptureCompleted(webrtc::DesktopFrame* frame) OVERRIDE;
  69
  70   // Chooses new output properties based on the supplied source size and the
  71   // properties requested to Allocate(), and dispatches OnFrameInfo[Changed]
  72   // notifications.
  73   void RefreshCaptureFormat(const webrtc::DesktopSize& frame_size);
  74
  75   // Method that is scheduled on |task_runner_| to be called on regular interval
  76   // to capture a frame.
  77   void OnCaptureTimer();
  78
  79   // Captures a frame and schedules timer for the next one.
  80   void CaptureFrameAndScheduleNext();
  81
  82   // Captures a single frame.
  83   void DoCapture();
  84
  85   // Task runner used for capturing operations.
  86   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  87
  88   // The underlying DesktopCapturer instance used to capture frames.
  89   scoped_ptr<webrtc::DesktopCapturer> desktop_capturer_;
  90
  91   // The device client which proxies device events to the controller. Accessed
  92   // on the task_runner_ thread.
  93   scoped_ptr<Client> client_;
  94
  95   // Requested video capture format (width, height, frame rate, etc).
  96   media::VideoCaptureParams requested_params_;
  97
  98   // Actual video capture format being generated.
  99   media::VideoCaptureFormat capture_format_;
 100
 101   // Size of frame most recently captured from the source.
 102   webrtc::DesktopSize previous_frame_size_;
 103
 104   // DesktopFrame into which captured frames are down-scaled and/or letterboxed,
 105   // depending upon the caller's requested capture capabilities. If frames can
 106   // be returned to the caller directly then this is NULL.
 107   scoped_ptr<webrtc::DesktopFrame> output_frame_;
 108
 109   // Sub-rectangle of |output_frame_| into which the source will be scaled
 110   // and/or letterboxed.
 111   webrtc::DesktopRect output_rect_;
 112
 113   // Timer used to capture the frame.
 114   base::OneShotTimer<Core> capture_timer_;
 115
 116   // True when waiting for |desktop_capturer_| to capture current frame.
 117   bool capture_in_progress_;
 118
 119   // True if the first capture call has returned. Used to log the first capture
 120   // result.
 121   bool first_capture_returned_;
 122
 123   // The type of the capturer.
 124   DesktopMediaID::Type capturer_type_;
 125
 126   scoped_ptr<webrtc::BasicDesktopFrame> black_frame_;
 127
 128   // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
 129   // screen from sleeping for the drive-by web.
 130   scoped_ptr<PowerSaveBlocker> power_save_blocker_;
 131
 132   DISALLOW_COPY_AND_ASSIGN(Core);
 133 };
 134
 135 DesktopCaptureDevice::Core::Core(
 136     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
 137     scoped_ptr<webrtc::DesktopCapturer> capturer,
 138     DesktopMediaID::Type type)
 139     : task_runner_(task_runner),
 140       desktop_capturer_(capturer.Pass()),
 141       capture_in_progress_(false),
 142       first_capture_returned_(false),
 143       capturer_type_(type) {
 144 }
 145
 146 DesktopCaptureDevice::Core::~Core() {
 147   DCHECK(task_runner_->BelongsToCurrentThread());
 148   client_.reset();
 149   output_frame_.reset();
 150   previous_frame_size_.set(0, 0);
 151   desktop_capturer_.reset();
 152 }
 153
 154 void DesktopCaptureDevice::Core::AllocateAndStart(
 155     const media::VideoCaptureParams& params,
 156     scoped_ptr<Client> client) {
 157   DCHECK(task_runner_->BelongsToCurrentThread());
 158   DCHECK_GT(params.requested_format.frame_size.GetArea(), 0);
 159   DCHECK_GT(params.requested_format.frame_rate, 0);
 160   DCHECK(desktop_capturer_);
 161   DCHECK(client.get());
 162   DCHECK(!client_.get());
 163
 164   client_ = client.Pass();
 165   requested_params_ = params;
 166
 167   capture_format_ = requested_params_.requested_format;
 168
 169   // This capturer always outputs ARGB, non-interlaced.
 170   capture_format_.pixel_format = media::PIXEL_FORMAT_ARGB;
 171
 172   power_save_blocker_.reset(PowerSaveBlocker::Create(
 173       PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
 174       "DesktopCaptureDevice is running").release());
 175
 176   desktop_capturer_->Start(this);
 177
 178   CaptureFrameAndScheduleNext();
 179 }
 180
 181 void DesktopCaptureDevice::Core::SetNotificationWindowId(
 182     gfx::NativeViewId window_id) {
 183   DCHECK(task_runner_->BelongsToCurrentThread());
 184   DCHECK(window_id);
 185   desktop_capturer_->SetExcludedWindow(window_id);
 186 }
 187
 188 webrtc::SharedMemory*
 189 DesktopCaptureDevice::Core::CreateSharedMemory(size_t size) {
 190   return NULL;
 191 }
 192
 193 void DesktopCaptureDevice::Core::OnCaptureCompleted(
 194     webrtc::DesktopFrame* frame) {
 195   DCHECK(task_runner_->BelongsToCurrentThread());
 196   DCHECK(capture_in_progress_);
 197
 198   if (!first_capture_returned_) {
 199     first_capture_returned_ = true;
 200     if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 201       IncrementDesktopCaptureCounter(frame ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 202                                            : FIRST_SCREEN_CAPTURE_FAILED);
 203     } else {
 204       IncrementDesktopCaptureCounter(frame ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 205                                            : FIRST_WINDOW_CAPTURE_FAILED);
 206     }
 207   }
 208
 209   capture_in_progress_ = false;
 210
 211   if (!frame) {
 212     std::string log("Failed to capture a frame.");
 213     LOG(ERROR) << log;
 214     client_->OnError(log);
 215     return;
 216   }
 217
 218   if (!client_)
 219     return;
 220
 221   base::TimeDelta capture_time(
 222       base::TimeDelta::FromMilliseconds(frame->capture_time_ms()));
 223   UMA_HISTOGRAM_TIMES(
 224       capturer_type_ == DesktopMediaID::TYPE_SCREEN ? kUmaScreenCaptureTime
 225                                                     : kUmaWindowCaptureTime,
 226       capture_time);
 227
 228   scoped_ptr<webrtc::DesktopFrame> owned_frame(frame);
 229
 230   // On OSX We receive a 1x1 frame when the shared window is minimized. It
 231   // cannot be subsampled to I420 and will be dropped downstream. So we replace
 232   // it with a black frame to avoid the video appearing frozen at the last
 233   // frame.
 234   if (frame->size().width() == 1 || frame->size().height() == 1) {
 235     if (!black_frame_.get()) {
 236       black_frame_.reset(
 237           new webrtc::BasicDesktopFrame(
 238               webrtc::DesktopSize(capture_format_.frame_size.width(),
 239                                   capture_format_.frame_size.height())));
 240       memset(black_frame_->data(),
 241              0,
 242              black_frame_->stride() * black_frame_->size().height());
 243     }
 244     owned_frame.reset();
 245     frame = black_frame_.get();
 246   }
 247
 248   // Handle initial frame size and size changes.
 249   RefreshCaptureFormat(frame->size());
 250
 251   webrtc::DesktopSize output_size(capture_format_.frame_size.width(),
 252                                   capture_format_.frame_size.height());
 253   size_t output_bytes = output_size.width() * output_size.height() *
 254       webrtc::DesktopFrame::kBytesPerPixel;
 255   const uint8_t* output_data = NULL;
 256   scoped_ptr<uint8_t[]> flipped_frame_buffer;
 257
 258   if (frame->size().equals(output_size)) {
 259     // If the captured frame matches the output size, we can return the pixel
 260     // data directly, without scaling.
 261     output_data = frame->data();
 262
 263     // If the |frame| generated by the screen capturer is inverted then we need
 264     // to flip |frame|.
 265     // This happens only on a specific platform. Refer to crbug.com/306876.
 266     if (frame->stride() < 0) {
 267       int height = frame->size().height();
 268       int bytes_per_row =
 269           frame->size().width() * webrtc::DesktopFrame::kBytesPerPixel;
 270       flipped_frame_buffer.reset(new uint8_t[output_bytes]);
 271       uint8_t* dest = flipped_frame_buffer.get();
 272       for (int row = 0; row < height; ++row) {
 273         memcpy(dest, output_data, bytes_per_row);
 274         dest += bytes_per_row;
 275         output_data += frame->stride();
 276       }
 277       output_data = flipped_frame_buffer.get();
 278     }
 279   } else {
 280     // Otherwise we need to down-scale and/or letterbox to the target format.
 281
 282     // Allocate a buffer of the correct size to scale the frame into.
 283     // |output_frame_| is cleared whenever |output_rect_| changes, so we don't
 284     // need to worry about clearing out stale pixel data in letterboxed areas.
 285     if (!output_frame_) {
 286       output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 287       memset(output_frame_->data(), 0, output_bytes);
 288     }
 289     DCHECK(output_frame_->size().equals(output_size));
 290
 291     // TODO(wez): Optimize this to scale only changed portions of the output,
 292     // using ARGBScaleClip().
 293     uint8_t* output_rect_data = output_frame_->data() +
 294         output_frame_->stride() * output_rect_.top() +
 295         webrtc::DesktopFrame::kBytesPerPixel * output_rect_.left();
 296     libyuv::ARGBScale(frame->data(), frame->stride(),
 297                       frame->size().width(), frame->size().height(),
 298                       output_rect_data, output_frame_->stride(),
 299                       output_rect_.width(), output_rect_.height(),
 300                       libyuv::kFilterBilinear);
 301     output_data = output_frame_->data();
 302   }
 303
 304   client_->OnIncomingCapturedData(
 305       output_data, output_bytes, capture_format_, 0, base::TimeTicks::Now());
 306 }
 307
 308 void DesktopCaptureDevice::Core::RefreshCaptureFormat(
 309     const webrtc::DesktopSize& frame_size) {
 310   if (previous_frame_size_.equals(frame_size))
 311     return;
 312
 313   // Clear the output frame, if any, since it will either need resizing, or
 314   // clearing of stale data in letterbox areas, anyway.
 315   output_frame_.reset();
 316
 317   if (previous_frame_size_.is_empty() ||
 318       requested_params_.allow_resolution_change) {
 319     // If this is the first frame, or the receiver supports variable resolution
 320     // then determine the output size by treating the requested width & height
 321     // as maxima.
 322     if (frame_size.width() >
 323             requested_params_.requested_format.frame_size.width() ||
 324         frame_size.height() >
 325             requested_params_.requested_format.frame_size.height()) {
 326       output_rect_ = ComputeLetterboxRect(
 327           webrtc::DesktopSize(
 328               requested_params_.requested_format.frame_size.width(),
 329               requested_params_.requested_format.frame_size.height()),
 330           frame_size);
 331       output_rect_.Translate(-output_rect_.left(), -output_rect_.top());
 332     } else {
 333       output_rect_ = webrtc::DesktopRect::MakeSize(frame_size);
 334     }
 335     capture_format_.frame_size.SetSize(output_rect_.width(),
 336                                        output_rect_.height());
 337   } else {
 338     // Otherwise the output frame size cannot change, so just scale and
 339     // letterbox.
 340     output_rect_ = ComputeLetterboxRect(
 341         webrtc::DesktopSize(capture_format_.frame_size.width(),
 342                             capture_format_.frame_size.height()),
 343         frame_size);
 344   }
 345
 346   previous_frame_size_ = frame_size;
 347 }
 348
 349 void DesktopCaptureDevice::Core::OnCaptureTimer() {
 350   DCHECK(task_runner_->BelongsToCurrentThread());
 351
 352   if (!client_)
 353     return;
 354
 355   CaptureFrameAndScheduleNext();
 356 }
 357
 358 void DesktopCaptureDevice::Core::CaptureFrameAndScheduleNext() {
 359   DCHECK(task_runner_->BelongsToCurrentThread());
 360
 361   base::TimeTicks started_time = base::TimeTicks::Now();
 362   DoCapture();
 363   base::TimeDelta last_capture_duration = base::TimeTicks::Now() - started_time;
 364
 365   // Limit frame-rate to reduce CPU consumption.
 366   base::TimeDelta capture_period = std::max(
 367     (last_capture_duration * 100) / kMaximumCpuConsumptionPercentage,
 368     base::TimeDelta::FromSeconds(1) / capture_format_.frame_rate);
 369
 370   // Schedule a task for the next frame.
 371   capture_timer_.Start(FROM_HERE, capture_period - last_capture_duration,
 372                        this, &Core::OnCaptureTimer);
 373 }
 374
 375 void DesktopCaptureDevice::Core::DoCapture() {
 376   DCHECK(task_runner_->BelongsToCurrentThread());
 377   DCHECK(!capture_in_progress_);
 378
 379   capture_in_progress_ = true;
 380   desktop_capturer_->Capture(webrtc::DesktopRegion());
 381
 382   // Currently only synchronous implementations of DesktopCapturer are
 383   // supported.
 384   DCHECK(!capture_in_progress_);
 385 }
 386
 387 // static
 388 scoped_ptr<media::VideoCaptureDevice> DesktopCaptureDevice::Create(
 389     const DesktopMediaID& source) {
 390   webrtc::DesktopCaptureOptions options =
 391       webrtc::DesktopCaptureOptions::CreateDefault();
 392   // Leave desktop effects enabled during WebRTC captures.
 393   options.set_disable_effects(false);
 394
 395   scoped_ptr<webrtc::DesktopCapturer> capturer;
 396
 397   switch (source.type) {
 398     case DesktopMediaID::TYPE_SCREEN: {
 399 #if defined(OS_WIN)
 400       options.set_allow_use_magnification_api(true);
 401 #endif
 402       scoped_ptr<webrtc::ScreenCapturer> screen_capturer(
 403           webrtc::ScreenCapturer::Create(options));
 404       if (screen_capturer && screen_capturer->SelectScreen(source.id)) {
 405         capturer.reset(new webrtc::DesktopAndCursorComposer(
 406             screen_capturer.release(),
 407             webrtc::MouseCursorMonitor::CreateForScreen(options, source.id)));
 408         IncrementDesktopCaptureCounter(SCREEN_CAPTURER_CREATED);
 409       }
 410       break;
 411     }
 412
 413     case DesktopMediaID::TYPE_WINDOW: {
 414       scoped_ptr<webrtc::WindowCapturer> window_capturer(
 415           webrtc::WindowCapturer::Create(options));
 416       if (window_capturer && window_capturer->SelectWindow(source.id)) {
 417         window_capturer->BringSelectedWindowToFront();
 418         capturer.reset(new webrtc::DesktopAndCursorComposer(
 419             window_capturer.release(),
 420             webrtc::MouseCursorMonitor::CreateForWindow(options, source.id)));
 421         IncrementDesktopCaptureCounter(WINDOW_CAPTURER_CREATED);
 422       }
 423       break;
 424     }
 425
 426     default: {
 427       NOTREACHED();
 428     }
 429   }
 430
 431   scoped_ptr<media::VideoCaptureDevice> result;
 432   if (capturer)
 433     result.reset(new DesktopCaptureDevice(capturer.Pass(), source.type));
 434
 435   return result.Pass();
 436 }
 437
 438 DesktopCaptureDevice::~DesktopCaptureDevice() {
 439   DCHECK(!core_);
 440 }
 441
 442 void DesktopCaptureDevice::AllocateAndStart(
 443     const media::VideoCaptureParams& params,
 444     scoped_ptr<Client> client) {
 445   thread_.message_loop_proxy()->PostTask(
 446       FROM_HERE,
 447       base::Bind(&Core::AllocateAndStart, base::Unretained(core_.get()), params,
 448                  base::Passed(&client)));
 449 }
 450
 451 void DesktopCaptureDevice::StopAndDeAllocate() {
 452   if (core_) {
 453     thread_.message_loop_proxy()->DeleteSoon(FROM_HERE, core_.release());
 454     thread_.Stop();
 455   }
 456 }
 457
 458 void DesktopCaptureDevice::SetNotificationWindowId(
 459     gfx::NativeViewId window_id) {
 460   thread_.message_loop_proxy()->PostTask(
 461       FROM_HERE,
 462       base::Bind(&Core::SetNotificationWindowId, base::Unretained(core_.get()),
 463                  window_id));
 464 }
 465
 466 DesktopCaptureDevice::DesktopCaptureDevice(
 467     scoped_ptr<webrtc::DesktopCapturer> capturer,
 468     DesktopMediaID::Type type)
 469     : thread_("desktopCaptureThread") {
 470 #if defined(OS_WIN)
 471   // On Windows the thread must be a UI thread.
 472   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_UI;
 473 #else
 474   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_DEFAULT;
 475 #endif
 476
 477   thread_.StartWithOptions(base::Thread::Options(thread_type, 0));
 478
 479   core_.reset(new Core(thread_.message_loop_proxy(), capturer.Pass(), type));
 480 }
 481
 482 }  // namespace content