remoting/client/plugin/pepper_view.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/plugin/pepper_view.h"
   6
   7 #include <functional>
   8
   9 #include "base/message_loop/message_loop.h"
  10 #include "base/strings/string_util.h"
  11 #include "base/synchronization/waitable_event.h"
  12 #include "base/time/time.h"
  13 #include "ppapi/cpp/completion_callback.h"
  14 #include "ppapi/cpp/image_data.h"
  15 #include "ppapi/cpp/point.h"
  16 #include "ppapi/cpp/rect.h"
  17 #include "ppapi/cpp/size.h"
  18 #include "remoting/base/util.h"
  19 #include "remoting/client/chromoting_stats.h"
  20 #include "remoting/client/client_context.h"
  21 #include "remoting/client/frame_producer.h"
  22 #include "remoting/client/plugin/chromoting_instance.h"
  23 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  24
  25 using base::Passed;
  26
  27 namespace {
  28
  29 // DesktopFrame that wraps a supplied pp::ImageData
  30 class PepperDesktopFrame : public webrtc::DesktopFrame {
  31  public:
  32   // Wraps the supplied ImageData.
  33   explicit PepperDesktopFrame(const pp::ImageData& buffer);
  34
  35   // Access to underlying pepper representation.
  36   const pp::ImageData& buffer() const {
  37     return buffer_;
  38   }
  39
  40  private:
  41   pp::ImageData buffer_;
  42 };
  43
  44 PepperDesktopFrame::PepperDesktopFrame(const pp::ImageData& buffer)
  45   : DesktopFrame(webrtc::DesktopSize(buffer.size().width(),
  46                                      buffer.size().height()),
  47                  buffer.stride(),
  48                  reinterpret_cast<uint8_t*>(buffer.data()),
  49                  NULL),
  50     buffer_(buffer) {}
  51
  52 }  // namespace
  53
  54 namespace remoting {
  55
  56 namespace {
  57
  58 // The maximum number of image buffers to be allocated at any point of time.
  59 const size_t kMaxPendingBuffersCount = 2;
  60
  61 }  // namespace
  62
  63 PepperView::PepperView(ChromotingInstance* instance,
  64                        ClientContext* context)
  65   : instance_(instance),
  66     context_(context),
  67     producer_(NULL),
  68     merge_buffer_(NULL),
  69     dips_to_device_scale_(1.0f),
  70     dips_to_view_scale_(1.0f),
  71     flush_pending_(false),
  72     is_initialized_(false),
  73     frame_received_(false),
  74     callback_factory_(this) {
  75 }
  76
  77 PepperView::~PepperView() {
  78   // The producer should now return any pending buffers. At this point, however,
  79   // ReturnBuffer() tasks scheduled by the producer will not be delivered,
  80   // so we free all the buffers once the producer's queue is empty.
  81   base::WaitableEvent done_event(true, false);
  82   producer_->RequestReturnBuffers(
  83       base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event)));
  84   done_event.Wait();
  85
  86   merge_buffer_ = NULL;
  87   while (!buffers_.empty()) {
  88     FreeBuffer(buffers_.front());
  89   }
  90 }
  91
  92 void PepperView::Initialize(FrameProducer* producer) {
  93   producer_ = producer;
  94   webrtc::DesktopFrame* buffer = AllocateBuffer();
  95   while (buffer) {
  96     producer_->DrawBuffer(buffer);
  97     buffer = AllocateBuffer();
  98   }
  99 }
 100
 101 void PepperView::SetView(const pp::View& view) {
 102   bool view_changed = false;
 103
 104   pp::Rect pp_size = view.GetRect();
 105   webrtc::DesktopSize new_dips_size(pp_size.width(), pp_size.height());
 106   float new_dips_to_device_scale = view.GetDeviceScale();
 107
 108   if (!dips_size_.equals(new_dips_size) ||
 109       dips_to_device_scale_ != new_dips_to_device_scale) {
 110     view_changed = true;
 111     dips_to_device_scale_ = new_dips_to_device_scale;
 112     dips_size_ = new_dips_size;
 113
 114     // If |dips_to_device_scale_| is > 1.0 then the device is high-DPI, and
 115     // there are actually |view_device_scale_| physical pixels for every one
 116     // Density Independent Pixel (DIP).  If we specify a scale of 1.0 to
 117     // Graphics2D then we can render at DIP resolution and let PPAPI up-scale
 118     // for high-DPI devices.
 119     dips_to_view_scale_ = 1.0f;
 120     view_size_ = dips_size_;
 121
 122     // If the view's DIP dimensions don't match the source then let the frame
 123     // producer do the scaling, and render at device resolution.
 124     if (!dips_size_.equals(source_size_)) {
 125       dips_to_view_scale_ = dips_to_device_scale_;
 126       view_size_.set(ceilf(dips_size_.width() * dips_to_view_scale_),
 127                      ceilf(dips_size_.height() * dips_to_view_scale_));
 128     }
 129
 130     // Create a 2D rendering context at the chosen frame dimensions.
 131     pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height());
 132     graphics2d_ = pp::Graphics2D(instance_, pp_size, false);
 133
 134     // Specify the scale from our coordinates to DIPs.
 135     graphics2d_.SetScale(1.0f / dips_to_view_scale_);
 136
 137     bool result = instance_->BindGraphics(graphics2d_);
 138
 139     // There is no good way to handle this error currently.
 140     DCHECK(result) << "Couldn't bind the device context.";
 141   }
 142
 143   // Ignore clip rectangle provided by the browser because it may not be
 144   // correct. See crbug.com/360240 . In case when the plugin is not visible
 145   // (e.g. another tab is selected) |clip_area_| is set to empty rectangle,
 146   // otherwise it's set to a rectangle that covers the whole plugin.
 147   //
 148   // TODO(sergeyu): Use view.GetClipRect() here after bug 360240 is fixed.
 149   webrtc::DesktopRect new_clip =
 150       view.IsVisible() ? webrtc::DesktopRect::MakeWH(
 151                              ceilf(pp_size.width() * dips_to_view_scale_),
 152                              ceilf(pp_size.height() * dips_to_view_scale_))
 153                        : webrtc::DesktopRect();
 154   if (!clip_area_.equals(new_clip)) {
 155     view_changed = true;
 156
 157     // YUV to RGB conversion may require even X and Y coordinates for
 158     // the top left corner of the clipping area.
 159     clip_area_ = AlignRect(new_clip);
 160     clip_area_.IntersectWith(webrtc::DesktopRect::MakeSize(view_size_));
 161   }
 162
 163   if (view_changed) {
 164     producer_->SetOutputSizeAndClip(view_size_, clip_area_);
 165     Initialize(producer_);
 166   }
 167 }
 168
 169 void PepperView::ApplyBuffer(const webrtc::DesktopSize& view_size,
 170                              const webrtc::DesktopRect& clip_area,
 171                              webrtc::DesktopFrame* buffer,
 172                              const webrtc::DesktopRegion& region,
 173                              const webrtc::DesktopRegion& shape) {
 174   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
 175
 176   if (!frame_received_) {
 177     instance_->OnFirstFrameReceived();
 178     frame_received_ = true;
 179   }
 180   // We cannot use the data in the buffer if its dimensions don't match the
 181   // current view size.
 182   // TODO(alexeypa): We could rescale and draw it (or even draw it without
 183   // rescaling) to reduce the perceived lag while we are waiting for
 184   // the properly scaled data.
 185   if (!view_size_.equals(view_size)) {
 186     FreeBuffer(buffer);
 187     Initialize(producer_);
 188   } else {
 189     FlushBuffer(clip_area, buffer, region);
 190     instance_->SetDesktopShape(shape);
 191   }
 192 }
 193
 194 void PepperView::ReturnBuffer(webrtc::DesktopFrame* buffer) {
 195   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
 196
 197   // Reuse the buffer if it is large enough, otherwise drop it on the floor
 198   // and allocate a new one.
 199   if (buffer->size().width() >= clip_area_.width() &&
 200       buffer->size().height() >= clip_area_.height()) {
 201     producer_->DrawBuffer(buffer);
 202   } else {
 203     FreeBuffer(buffer);
 204     Initialize(producer_);
 205   }
 206 }
 207
 208 void PepperView::SetSourceSize(const webrtc::DesktopSize& source_size,
 209                                const webrtc::DesktopVector& source_dpi) {
 210   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
 211
 212   if (source_size_.equals(source_size) && source_dpi_.equals(source_dpi))
 213     return;
 214
 215   source_size_ = source_size;
 216   source_dpi_ = source_dpi;
 217
 218   // Notify JavaScript of the change in source size.
 219   instance_->SetDesktopSize(source_size, source_dpi);
 220 }
 221
 222 FrameConsumer::PixelFormat PepperView::GetPixelFormat() {
 223   return FORMAT_BGRA;
 224 }
 225
 226 webrtc::DesktopFrame* PepperView::AllocateBuffer() {
 227   if (buffers_.size() >= kMaxPendingBuffersCount)
 228     return NULL;
 229
 230   if (clip_area_.width()==0 || clip_area_.height()==0)
 231     return NULL;
 232
 233   // Create an image buffer of the required size, but don't zero it.
 234   pp::ImageData buffer_data(instance_,
 235                     PP_IMAGEDATAFORMAT_BGRA_PREMUL,
 236                     pp::Size(clip_area_.width(),
 237                              clip_area_.height()),
 238                     false);
 239   if (buffer_data.is_null()) {
 240     LOG(WARNING) << "Not enough memory for frame buffers.";
 241     return NULL;
 242   }
 243
 244   webrtc::DesktopFrame* buffer = new PepperDesktopFrame(buffer_data);
 245   buffers_.push_back(buffer);
 246   return buffer;
 247 }
 248
 249 void PepperView::FreeBuffer(webrtc::DesktopFrame* buffer) {
 250   DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end());
 251
 252   buffers_.remove(buffer);
 253   delete buffer;
 254 }
 255
 256 void PepperView::FlushBuffer(const webrtc::DesktopRect& clip_area,
 257                              webrtc::DesktopFrame* buffer,
 258                              const webrtc::DesktopRegion& region) {
 259   // Defer drawing if the flush is already in progress.
 260   if (flush_pending_) {
 261     // |merge_buffer_| is guaranteed to be free here because we allocate only
 262     // two buffers simultaneously. If more buffers are allowed this code should
 263     // apply all pending changes to the screen.
 264     DCHECK(merge_buffer_ == NULL);
 265
 266     merge_clip_area_ = clip_area;
 267     merge_buffer_ = buffer;
 268     merge_region_ = region;
 269     return;
 270   }
 271
 272   // Notify Pepper API about the updated areas and flush pixels to the screen.
 273   base::Time start_time = base::Time::Now();
 274
 275   for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) {
 276     webrtc::DesktopRect rect = i.rect();
 277
 278     // Re-clip |region| with the current clipping area |clip_area_| because
 279     // the latter could change from the time the buffer was drawn.
 280     rect.IntersectWith(clip_area_);
 281     if (rect.is_empty())
 282       continue;
 283
 284     // Specify the rectangle coordinates relative to the clipping area.
 285     rect.Translate(-clip_area.left(), -clip_area.top());
 286
 287     // Pepper Graphics 2D has a strange and badly documented API that the
 288     // point here is the offset from the source rect. Why?
 289     graphics2d_.PaintImageData(
 290         static_cast<PepperDesktopFrame*>(buffer)->buffer(),
 291         pp::Point(clip_area.left(), clip_area.top()),
 292         pp::Rect(rect.left(), rect.top(), rect.width(), rect.height()));
 293   }
 294
 295   // Notify the producer that some parts of the region weren't painted because
 296   // the clipping area has changed already.
 297   if (!clip_area.equals(clip_area_)) {
 298     webrtc::DesktopRegion not_painted = region;
 299     not_painted.Subtract(clip_area_);
 300     if (!not_painted.is_empty()) {
 301       producer_->InvalidateRegion(not_painted);
 302     }
 303   }
 304
 305   // Flush the updated areas to the screen.
 306   pp::CompletionCallback callback =
 307       callback_factory_.NewCallback(&PepperView::OnFlushDone,
 308                                     start_time,
 309                                     buffer);
 310   int error = graphics2d_.Flush(callback);
 311   CHECK(error == PP_OK_COMPLETIONPENDING);
 312   flush_pending_ = true;
 313 }
 314
 315 void PepperView::OnFlushDone(int result,
 316                              const base::Time& paint_start,
 317                              webrtc::DesktopFrame* buffer) {
 318   DCHECK(context_->main_task_runner()->BelongsToCurrentThread());
 319   DCHECK(flush_pending_);
 320
 321   instance_->GetStats()->video_paint_ms()->Record(
 322       (base::Time::Now() - paint_start).InMilliseconds());
 323
 324   flush_pending_ = false;
 325   ReturnBuffer(buffer);
 326
 327   // If there is a buffer queued for rendering then render it now.
 328   if (merge_buffer_ != NULL) {
 329     buffer = merge_buffer_;
 330     merge_buffer_ = NULL;
 331     FlushBuffer(merge_clip_area_, buffer, merge_region_);
 332   }
 333 }
 334
 335 }  // namespace remoting