content/browser/media/capture/desktop_capture_device.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/browser/media/capture/desktop_capture_device.h"
   6
   7 #include "base/bind.h"
   8 #include "base/location.h"
   9 #include "base/logging.h"
  10 #include "base/metrics/histogram.h"
  11 #include "base/strings/string_number_conversions.h"
  12 #include "base/synchronization/lock.h"
  13 #include "base/threading/thread.h"
  14 #include "base/timer/timer.h"
  15 #include "content/browser/media/capture/desktop_capture_device_uma_types.h"
  16 #include "content/public/browser/browser_thread.h"
  17 #include "content/public/browser/desktop_media_id.h"
  18 #include "content/public/browser/power_save_blocker.h"
  19 #include "media/base/video_util.h"
  20 #include "third_party/libyuv/include/libyuv/scale_argb.h"
  21 #include "third_party/webrtc/modules/desktop_capture/cropping_window_capturer.h"
  22 #include "third_party/webrtc/modules/desktop_capture/desktop_and_cursor_composer.h"
  23 #include "third_party/webrtc/modules/desktop_capture/desktop_capture_options.h"
  24 #include "third_party/webrtc/modules/desktop_capture/desktop_capturer.h"
  25 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  26 #include "third_party/webrtc/modules/desktop_capture/mouse_cursor_monitor.h"
  27 #include "third_party/webrtc/modules/desktop_capture/screen_capturer.h"
  28
  29 namespace content {
  30
  31 namespace {
  32
  33 // Maximum CPU time percentage of a single core that can be consumed for desktop
  34 // capturing. This means that on systems where screen scraping is slow we may
  35 // need to capture at frame rate lower than requested. This is necessary to keep
  36 // UI responsive.
  37 const int kMaximumCpuConsumptionPercentage = 50;
  38
  39 webrtc::DesktopRect ComputeLetterboxRect(
  40     const webrtc::DesktopSize& max_size,
  41     const webrtc::DesktopSize& source_size) {
  42   gfx::Rect result = media::ComputeLetterboxRegion(
  43       gfx::Rect(0, 0, max_size.width(), max_size.height()),
  44       gfx::Size(source_size.width(), source_size.height()));
  45   return webrtc::DesktopRect::MakeLTRB(
  46       result.x(), result.y(), result.right(), result.bottom());
  47 }
  48
  49 bool IsFrameUnpackedOrInverted(webrtc::DesktopFrame* frame) {
  50   return frame->stride() !=
  51       frame->size().width() * webrtc::DesktopFrame::kBytesPerPixel;
  52 }
  53
  54 }  // namespace
  55
  56 class DesktopCaptureDevice::Core : public webrtc::DesktopCapturer::Callback {
  57  public:
  58   Core(scoped_refptr<base::SingleThreadTaskRunner> task_runner,
  59        scoped_ptr<webrtc::DesktopCapturer> capturer,
  60        DesktopMediaID::Type type);
  61   ~Core() override;
  62
  63   // Implementation of VideoCaptureDevice methods.
  64   void AllocateAndStart(const media::VideoCaptureParams& params,
  65                         scoped_ptr<Client> client);
  66
  67   void SetNotificationWindowId(gfx::NativeViewId window_id);
  68
  69  private:
  70
  71   // webrtc::DesktopCapturer::Callback interface
  72   webrtc::SharedMemory* CreateSharedMemory(size_t size) override;
  73   void OnCaptureCompleted(webrtc::DesktopFrame* frame) override;
  74
  75   // Chooses new output properties based on the supplied source size and the
  76   // properties requested to Allocate(), and dispatches OnFrameInfo[Changed]
  77   // notifications.
  78   void RefreshCaptureFormat(const webrtc::DesktopSize& frame_size);
  79
  80   // Method that is scheduled on |task_runner_| to be called on regular interval
  81   // to capture a frame.
  82   void OnCaptureTimer();
  83
  84   // Captures a frame and schedules timer for the next one.
  85   void CaptureFrameAndScheduleNext();
  86
  87   // Captures a single frame.
  88   void DoCapture();
  89
  90   // Task runner used for capturing operations.
  91   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  92
  93   // The underlying DesktopCapturer instance used to capture frames.
  94   scoped_ptr<webrtc::DesktopCapturer> desktop_capturer_;
  95
  96   // The device client which proxies device events to the controller. Accessed
  97   // on the task_runner_ thread.
  98   scoped_ptr<Client> client_;
  99
 100   // Requested video capture format (width, height, frame rate, etc).
 101   media::VideoCaptureParams requested_params_;
 102
 103   // Actual video capture format being generated.
 104   media::VideoCaptureFormat capture_format_;
 105
 106   // Size of frame most recently captured from the source.
 107   webrtc::DesktopSize previous_frame_size_;
 108
 109   // DesktopFrame into which captured frames are down-scaled and/or letterboxed,
 110   // depending upon the caller's requested capture capabilities. If frames can
 111   // be returned to the caller directly then this is NULL.
 112   scoped_ptr<webrtc::DesktopFrame> output_frame_;
 113
 114   // Sub-rectangle of |output_frame_| into which the source will be scaled
 115   // and/or letterboxed.
 116   webrtc::DesktopRect output_rect_;
 117
 118   // Timer used to capture the frame.
 119   base::OneShotTimer<Core> capture_timer_;
 120
 121   // True when waiting for |desktop_capturer_| to capture current frame.
 122   bool capture_in_progress_;
 123
 124   // True if the first capture call has returned. Used to log the first capture
 125   // result.
 126   bool first_capture_returned_;
 127
 128   // The type of the capturer.
 129   DesktopMediaID::Type capturer_type_;
 130
 131   scoped_ptr<webrtc::BasicDesktopFrame> black_frame_;
 132
 133   // TODO(jiayl): Remove power_save_blocker_ when there is an API to keep the
 134   // screen from sleeping for the drive-by web.
 135   scoped_ptr<PowerSaveBlocker> power_save_blocker_;
 136
 137   DISALLOW_COPY_AND_ASSIGN(Core);
 138 };
 139
 140 DesktopCaptureDevice::Core::Core(
 141     scoped_refptr<base::SingleThreadTaskRunner> task_runner,
 142     scoped_ptr<webrtc::DesktopCapturer> capturer,
 143     DesktopMediaID::Type type)
 144     : task_runner_(task_runner),
 145       desktop_capturer_(capturer.Pass()),
 146       capture_in_progress_(false),
 147       first_capture_returned_(false),
 148       capturer_type_(type) {
 149 }
 150
 151 DesktopCaptureDevice::Core::~Core() {
 152   DCHECK(task_runner_->BelongsToCurrentThread());
 153   client_.reset();
 154   output_frame_.reset();
 155   previous_frame_size_.set(0, 0);
 156   desktop_capturer_.reset();
 157 }
 158
 159 void DesktopCaptureDevice::Core::AllocateAndStart(
 160     const media::VideoCaptureParams& params,
 161     scoped_ptr<Client> client) {
 162   DCHECK(task_runner_->BelongsToCurrentThread());
 163   DCHECK_GT(params.requested_format.frame_size.GetArea(), 0);
 164   DCHECK_GT(params.requested_format.frame_rate, 0);
 165   DCHECK(desktop_capturer_);
 166   DCHECK(client.get());
 167   DCHECK(!client_.get());
 168
 169   client_ = client.Pass();
 170   requested_params_ = params;
 171
 172   capture_format_ = requested_params_.requested_format;
 173
 174   // This capturer always outputs ARGB, non-interlaced.
 175   capture_format_.pixel_format = media::PIXEL_FORMAT_ARGB;
 176
 177   power_save_blocker_.reset(
 178       PowerSaveBlocker::Create(
 179           PowerSaveBlocker::kPowerSaveBlockPreventDisplaySleep,
 180           PowerSaveBlocker::kReasonOther,
 181           "DesktopCaptureDevice is running").release());
 182
 183   desktop_capturer_->Start(this);
 184
 185   CaptureFrameAndScheduleNext();
 186 }
 187
 188 void DesktopCaptureDevice::Core::SetNotificationWindowId(
 189     gfx::NativeViewId window_id) {
 190   DCHECK(task_runner_->BelongsToCurrentThread());
 191   DCHECK(window_id);
 192   desktop_capturer_->SetExcludedWindow(window_id);
 193 }
 194
 195 webrtc::SharedMemory*
 196 DesktopCaptureDevice::Core::CreateSharedMemory(size_t size) {
 197   return NULL;
 198 }
 199
 200 void DesktopCaptureDevice::Core::OnCaptureCompleted(
 201     webrtc::DesktopFrame* frame) {
 202   DCHECK(task_runner_->BelongsToCurrentThread());
 203   DCHECK(capture_in_progress_);
 204
 205   if (!first_capture_returned_) {
 206     first_capture_returned_ = true;
 207     if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 208       IncrementDesktopCaptureCounter(frame ? FIRST_SCREEN_CAPTURE_SUCCEEDED
 209                                            : FIRST_SCREEN_CAPTURE_FAILED);
 210     } else {
 211       IncrementDesktopCaptureCounter(frame ? FIRST_WINDOW_CAPTURE_SUCCEEDED
 212                                            : FIRST_WINDOW_CAPTURE_FAILED);
 213     }
 214   }
 215
 216   capture_in_progress_ = false;
 217
 218   if (!frame) {
 219     std::string log("Failed to capture a frame.");
 220     LOG(ERROR) << log;
 221     client_->OnError(log);
 222     return;
 223   }
 224
 225   if (!client_)
 226     return;
 227
 228   base::TimeDelta capture_time(
 229       base::TimeDelta::FromMilliseconds(frame->capture_time_ms()));
 230
 231   // The two UMA_ blocks must be put in its own scope since it creates a static
 232   // variable which expected constant histogram name.
 233   if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) {
 234     UMA_HISTOGRAM_TIMES(kUmaScreenCaptureTime, capture_time);
 235   } else {
 236     UMA_HISTOGRAM_TIMES(kUmaWindowCaptureTime, capture_time);
 237   }
 238
 239   scoped_ptr<webrtc::DesktopFrame> owned_frame(frame);
 240
 241   // On OSX We receive a 1x1 frame when the shared window is minimized. It
 242   // cannot be subsampled to I420 and will be dropped downstream. So we replace
 243   // it with a black frame to avoid the video appearing frozen at the last
 244   // frame.
 245   if (frame->size().width() == 1 || frame->size().height() == 1) {
 246     if (!black_frame_.get()) {
 247       black_frame_.reset(
 248           new webrtc::BasicDesktopFrame(
 249               webrtc::DesktopSize(capture_format_.frame_size.width(),
 250                                   capture_format_.frame_size.height())));
 251       memset(black_frame_->data(),
 252              0,
 253              black_frame_->stride() * black_frame_->size().height());
 254     }
 255     owned_frame.reset();
 256     frame = black_frame_.get();
 257   }
 258
 259   // Handle initial frame size and size changes.
 260   RefreshCaptureFormat(frame->size());
 261
 262   webrtc::DesktopSize output_size(capture_format_.frame_size.width(),
 263                                   capture_format_.frame_size.height());
 264   size_t output_bytes = output_size.width() * output_size.height() *
 265       webrtc::DesktopFrame::kBytesPerPixel;
 266   const uint8_t* output_data = NULL;
 267
 268   if (!frame->size().equals(output_size)) {
 269     // Down-scale and/or letterbox to the target format if the frame does not
 270     // match the output size.
 271
 272     // Allocate a buffer of the correct size to scale the frame into.
 273     // |output_frame_| is cleared whenever |output_rect_| changes, so we don't
 274     // need to worry about clearing out stale pixel data in letterboxed areas.
 275     if (!output_frame_) {
 276       output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 277       memset(output_frame_->data(), 0, output_bytes);
 278     }
 279     DCHECK(output_frame_->size().equals(output_size));
 280
 281     // TODO(wez): Optimize this to scale only changed portions of the output,
 282     // using ARGBScaleClip().
 283     uint8_t* output_rect_data = output_frame_->data() +
 284         output_frame_->stride() * output_rect_.top() +
 285         webrtc::DesktopFrame::kBytesPerPixel * output_rect_.left();
 286     libyuv::ARGBScale(frame->data(), frame->stride(),
 287                       frame->size().width(), frame->size().height(),
 288                       output_rect_data, output_frame_->stride(),
 289                       output_rect_.width(), output_rect_.height(),
 290                       libyuv::kFilterBilinear);
 291     output_data = output_frame_->data();
 292   } else if (IsFrameUnpackedOrInverted(frame)) {
 293     // If |frame| is not packed top-to-bottom then create a packed top-to-bottom
 294     // copy.
 295     // This is required if the frame is inverted (see crbug.com/306876), or if
 296     // |frame| is cropped form a larger frame (see crbug.com/437740).
 297     if (!output_frame_) {
 298       output_frame_.reset(new webrtc::BasicDesktopFrame(output_size));
 299       memset(output_frame_->data(), 0, output_bytes);
 300     }
 301
 302     output_frame_->CopyPixelsFrom(
 303         *frame,
 304         webrtc::DesktopVector(),
 305         webrtc::DesktopRect::MakeSize(frame->size()));
 306     output_data = output_frame_->data();
 307   } else {
 308     // If the captured frame matches the output size, we can return the pixel
 309     // data directly.
 310     output_data = frame->data();
 311   }
 312
 313   client_->OnIncomingCapturedData(
 314       output_data, output_bytes, capture_format_, 0, base::TimeTicks::Now());
 315 }
 316
 317 void DesktopCaptureDevice::Core::RefreshCaptureFormat(
 318     const webrtc::DesktopSize& frame_size) {
 319   if (previous_frame_size_.equals(frame_size))
 320     return;
 321
 322   // Clear the output frame, if any, since it will either need resizing, or
 323   // clearing of stale data in letterbox areas, anyway.
 324   output_frame_.reset();
 325
 326   if (previous_frame_size_.is_empty() ||
 327       requested_params_.resolution_change_policy ==
 328       media::RESOLUTION_POLICY_DYNAMIC_WITHIN_LIMIT) {
 329     // If this is the first frame, or the receiver supports variable resolution
 330     // then determine the output size by treating the requested width & height
 331     // as maxima.
 332     if (frame_size.width() >
 333             requested_params_.requested_format.frame_size.width() ||
 334         frame_size.height() >
 335             requested_params_.requested_format.frame_size.height()) {
 336       output_rect_ = ComputeLetterboxRect(
 337           webrtc::DesktopSize(
 338               requested_params_.requested_format.frame_size.width(),
 339               requested_params_.requested_format.frame_size.height()),
 340           frame_size);
 341       output_rect_.Translate(-output_rect_.left(), -output_rect_.top());
 342     } else {
 343       output_rect_ = webrtc::DesktopRect::MakeSize(frame_size);
 344     }
 345     capture_format_.frame_size.SetSize(output_rect_.width(),
 346                                        output_rect_.height());
 347   } else {
 348     // Otherwise the output frame size cannot change, so just scale and
 349     // letterbox.
 350     output_rect_ = ComputeLetterboxRect(
 351         webrtc::DesktopSize(capture_format_.frame_size.width(),
 352                             capture_format_.frame_size.height()),
 353         frame_size);
 354   }
 355
 356   previous_frame_size_ = frame_size;
 357 }
 358
 359 void DesktopCaptureDevice::Core::OnCaptureTimer() {
 360   DCHECK(task_runner_->BelongsToCurrentThread());
 361
 362   if (!client_)
 363     return;
 364
 365   CaptureFrameAndScheduleNext();
 366 }
 367
 368 void DesktopCaptureDevice::Core::CaptureFrameAndScheduleNext() {
 369   DCHECK(task_runner_->BelongsToCurrentThread());
 370
 371   base::TimeTicks started_time = base::TimeTicks::Now();
 372   DoCapture();
 373   base::TimeDelta last_capture_duration = base::TimeTicks::Now() - started_time;
 374
 375   // Limit frame-rate to reduce CPU consumption.
 376   base::TimeDelta capture_period = std::max(
 377     (last_capture_duration * 100) / kMaximumCpuConsumptionPercentage,
 378     base::TimeDelta::FromSeconds(1) / capture_format_.frame_rate);
 379
 380   // Schedule a task for the next frame.
 381   capture_timer_.Start(FROM_HERE, capture_period - last_capture_duration,
 382                        this, &Core::OnCaptureTimer);
 383 }
 384
 385 void DesktopCaptureDevice::Core::DoCapture() {
 386   DCHECK(task_runner_->BelongsToCurrentThread());
 387   DCHECK(!capture_in_progress_);
 388
 389   capture_in_progress_ = true;
 390   desktop_capturer_->Capture(webrtc::DesktopRegion());
 391
 392   // Currently only synchronous implementations of DesktopCapturer are
 393   // supported.
 394   DCHECK(!capture_in_progress_);
 395 }
 396
 397 // static
 398 scoped_ptr<media::VideoCaptureDevice> DesktopCaptureDevice::Create(
 399     const DesktopMediaID& source) {
 400   webrtc::DesktopCaptureOptions options =
 401       webrtc::DesktopCaptureOptions::CreateDefault();
 402   // Leave desktop effects enabled during WebRTC captures.
 403   options.set_disable_effects(false);
 404
 405 #if defined(OS_WIN)
 406       options.set_allow_use_magnification_api(true);
 407 #endif
 408
 409   scoped_ptr<webrtc::DesktopCapturer> capturer;
 410
 411   switch (source.type) {
 412     case DesktopMediaID::TYPE_SCREEN: {
 413       scoped_ptr<webrtc::ScreenCapturer> screen_capturer(
 414           webrtc::ScreenCapturer::Create(options));
 415       if (screen_capturer && screen_capturer->SelectScreen(source.id)) {
 416         capturer.reset(new webrtc::DesktopAndCursorComposer(
 417             screen_capturer.release(),
 418             webrtc::MouseCursorMonitor::CreateForScreen(options, source.id)));
 419         IncrementDesktopCaptureCounter(SCREEN_CAPTURER_CREATED);
 420       }
 421       break;
 422     }
 423
 424     case DesktopMediaID::TYPE_WINDOW: {
 425       scoped_ptr<webrtc::WindowCapturer> window_capturer(
 426           webrtc::CroppingWindowCapturer::Create(options));
 427       if (window_capturer && window_capturer->SelectWindow(source.id)) {
 428         window_capturer->BringSelectedWindowToFront();
 429         capturer.reset(new webrtc::DesktopAndCursorComposer(
 430             window_capturer.release(),
 431             webrtc::MouseCursorMonitor::CreateForWindow(options, source.id)));
 432         IncrementDesktopCaptureCounter(WINDOW_CAPTURER_CREATED);
 433       }
 434       break;
 435     }
 436
 437     default: {
 438       NOTREACHED();
 439     }
 440   }
 441
 442   scoped_ptr<media::VideoCaptureDevice> result;
 443   if (capturer)
 444     result.reset(new DesktopCaptureDevice(capturer.Pass(), source.type));
 445
 446   return result.Pass();
 447 }
 448
 449 DesktopCaptureDevice::~DesktopCaptureDevice() {
 450   DCHECK(!core_);
 451 }
 452
 453 void DesktopCaptureDevice::AllocateAndStart(
 454     const media::VideoCaptureParams& params,
 455     scoped_ptr<Client> client) {
 456   thread_.task_runner()->PostTask(
 457       FROM_HERE,
 458       base::Bind(&Core::AllocateAndStart, base::Unretained(core_.get()), params,
 459                  base::Passed(&client)));
 460 }
 461
 462 void DesktopCaptureDevice::StopAndDeAllocate() {
 463   if (core_) {
 464     thread_.task_runner()->DeleteSoon(FROM_HERE, core_.release());
 465     thread_.Stop();
 466   }
 467 }
 468
 469 void DesktopCaptureDevice::SetNotificationWindowId(
 470     gfx::NativeViewId window_id) {
 471   // This may be called after the capturer has been stopped.
 472   if (!core_)
 473     return;
 474   thread_.task_runner()->PostTask(
 475       FROM_HERE,
 476       base::Bind(&Core::SetNotificationWindowId,
 477                  base::Unretained(core_.get()),
 478                  window_id));
 479 }
 480
 481 DesktopCaptureDevice::DesktopCaptureDevice(
 482     scoped_ptr<webrtc::DesktopCapturer> capturer,
 483     DesktopMediaID::Type type)
 484     : thread_("desktopCaptureThread") {
 485 #if defined(OS_WIN)
 486   // On Windows the thread must be a UI thread.
 487   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_UI;
 488 #else
 489   base::MessageLoop::Type thread_type = base::MessageLoop::TYPE_DEFAULT;
 490 #endif
 491
 492   thread_.StartWithOptions(base::Thread::Options(thread_type, 0));
 493
 494   core_.reset(new Core(thread_.task_runner(), capturer.Pass(), type));
 495 }
 496
 497 }  // namespace content