remoting/client/plugin/pepper_video_renderer_2d.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/plugin/pepper_video_renderer_2d.h"
   6
   7 #include <functional>
   8
   9 #include "base/bind.h"
  10 #include "base/strings/string_util.h"
  11 #include "base/synchronization/waitable_event.h"
  12 #include "base/time/time.h"
  13 #include "ppapi/cpp/completion_callback.h"
  14 #include "ppapi/cpp/image_data.h"
  15 #include "ppapi/cpp/instance.h"
  16 #include "ppapi/cpp/point.h"
  17 #include "ppapi/cpp/rect.h"
  18 #include "ppapi/cpp/size.h"
  19 #include "remoting/base/util.h"
  20 #include "remoting/client/chromoting_stats.h"
  21 #include "remoting/client/client_context.h"
  22 #include "remoting/client/frame_consumer_proxy.h"
  23 #include "remoting/client/frame_producer.h"
  24 #include "remoting/client/software_video_renderer.h"
  25 #include "remoting/proto/video.pb.h"
  26 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  27
  28 namespace {
  29
  30 // DesktopFrame that wraps a supplied pp::ImageData
  31 class PepperDesktopFrame : public webrtc::DesktopFrame {
  32  public:
  33   // Wraps the supplied ImageData.
  34   explicit PepperDesktopFrame(const pp::ImageData& buffer);
  35
  36   // Access to underlying pepper representation.
  37   const pp::ImageData& buffer() const {
  38     return buffer_;
  39   }
  40
  41  private:
  42   pp::ImageData buffer_;
  43 };
  44
  45 PepperDesktopFrame::PepperDesktopFrame(const pp::ImageData& buffer)
  46   : DesktopFrame(webrtc::DesktopSize(buffer.size().width(),
  47                                      buffer.size().height()),
  48                  buffer.stride(),
  49                  reinterpret_cast<uint8_t*>(buffer.data()),
  50                  nullptr),
  51     buffer_(buffer) {}
  52
  53 }  // namespace
  54
  55 namespace remoting {
  56
  57 namespace {
  58
  59 // The maximum number of image buffers to be allocated at any point of time.
  60 const size_t kMaxPendingBuffersCount = 2;
  61
  62 }  // namespace
  63
  64 PepperVideoRenderer2D::PepperVideoRenderer2D()
  65     : instance_(nullptr),
  66       event_handler_(nullptr),
  67       merge_buffer_(nullptr),
  68       dips_to_device_scale_(1.0f),
  69       dips_to_view_scale_(1.0f),
  70       flush_pending_(false),
  71       frame_received_(false),
  72       debug_dirty_region_(false),
  73       callback_factory_(this),
  74       weak_factory_(this) {
  75 }
  76
  77 PepperVideoRenderer2D::~PepperVideoRenderer2D() {
  78   // The producer should now return any pending buffers. At this point, however,
  79   // ReturnBuffer() tasks scheduled by the producer will not be delivered,
  80   // so we free all the buffers once the producer's queue is empty.
  81   base::WaitableEvent done_event(true, false);
  82   software_video_renderer_->RequestReturnBuffers(
  83       base::Bind(&base::WaitableEvent::Signal, base::Unretained(&done_event)));
  84   done_event.Wait();
  85
  86   merge_buffer_ = nullptr;
  87   while (!buffers_.empty()) {
  88     FreeBuffer(buffers_.front());
  89   }
  90 }
  91
  92 bool PepperVideoRenderer2D::Initialize(pp::Instance* instance,
  93                                        const ClientContext& context,
  94                                        EventHandler* event_handler) {
  95   DCHECK(CalledOnValidThread());
  96   DCHECK(!instance_);
  97   DCHECK(!event_handler_);
  98   DCHECK(instance);
  99   DCHECK(event_handler);
 100
 101   instance_ = instance;
 102   event_handler_ = event_handler;
 103   frame_consumer_proxy_ = new FrameConsumerProxy(
 104       context.main_task_runner(), weak_factory_.GetWeakPtr());
 105   software_video_renderer_.reset(new SoftwareVideoRenderer(
 106       context.main_task_runner(), context.decode_task_runner(),
 107       frame_consumer_proxy_));
 108
 109   return true;
 110 }
 111
 112 void PepperVideoRenderer2D::OnViewChanged(const pp::View& view) {
 113   DCHECK(CalledOnValidThread());
 114
 115   bool view_changed = false;
 116
 117   pp::Rect pp_size = view.GetRect();
 118   webrtc::DesktopSize new_dips_size(pp_size.width(), pp_size.height());
 119   float new_dips_to_device_scale = view.GetDeviceScale();
 120
 121   if (!dips_size_.equals(new_dips_size) ||
 122       dips_to_device_scale_ != new_dips_to_device_scale) {
 123     view_changed = true;
 124     dips_to_device_scale_ = new_dips_to_device_scale;
 125     dips_size_ = new_dips_size;
 126
 127     // If |dips_to_device_scale_| is > 1.0 then the device is high-DPI, and
 128     // there are actually |view_device_scale_| physical pixels for every one
 129     // Density Independent Pixel (DIP).  If we specify a scale of 1.0 to
 130     // Graphics2D then we can render at DIP resolution and let PPAPI up-scale
 131     // for high-DPI devices.
 132     dips_to_view_scale_ = 1.0f;
 133     view_size_ = dips_size_;
 134
 135     // If the view's DIP dimensions don't match the source then let the frame
 136     // producer do the scaling, and render at device resolution.
 137     if (!dips_size_.equals(source_size_)) {
 138       dips_to_view_scale_ = dips_to_device_scale_;
 139       view_size_.set(ceilf(dips_size_.width() * dips_to_view_scale_),
 140                      ceilf(dips_size_.height() * dips_to_view_scale_));
 141     }
 142
 143     // Create a 2D rendering context at the chosen frame dimensions.
 144     pp::Size pp_size = pp::Size(view_size_.width(), view_size_.height());
 145     graphics2d_ = pp::Graphics2D(instance_, pp_size, false);
 146
 147     // Specify the scale from our coordinates to DIPs.
 148     graphics2d_.SetScale(1.0f / dips_to_view_scale_);
 149
 150     bool result = instance_->BindGraphics(graphics2d_);
 151
 152     // There is no good way to handle this error currently.
 153     DCHECK(result) << "Couldn't bind the device context.";
 154   }
 155
 156   // Ignore clip rectangle provided by the browser because it may not be
 157   // correct. See crbug.com/360240 . In case when the plugin is not visible
 158   // (e.g. another tab is selected) |clip_area_| is set to empty rectangle,
 159   // otherwise it's set to a rectangle that covers the whole plugin.
 160   //
 161   // TODO(sergeyu): Use view.GetClipRect() here after bug 360240 is fixed.
 162   webrtc::DesktopRect new_clip =
 163       view.IsVisible() ? webrtc::DesktopRect::MakeWH(
 164                              ceilf(pp_size.width() * dips_to_view_scale_),
 165                              ceilf(pp_size.height() * dips_to_view_scale_))
 166                        : webrtc::DesktopRect();
 167   if (!clip_area_.equals(new_clip)) {
 168     view_changed = true;
 169
 170     // YUV to RGB conversion may require even X and Y coordinates for
 171     // the top left corner of the clipping area.
 172     clip_area_ = AlignRect(new_clip);
 173     clip_area_.IntersectWith(webrtc::DesktopRect::MakeSize(view_size_));
 174   }
 175
 176   if (view_changed) {
 177     software_video_renderer_->SetOutputSizeAndClip(view_size_, clip_area_);
 178     AllocateBuffers();
 179   }
 180 }
 181
 182 void PepperVideoRenderer2D::EnableDebugDirtyRegion(bool enable) {
 183   debug_dirty_region_ = enable;
 184 }
 185
 186 void PepperVideoRenderer2D::OnSessionConfig(
 187     const protocol::SessionConfig& config) {
 188   DCHECK(CalledOnValidThread());
 189
 190   software_video_renderer_->OnSessionConfig(config);
 191   AllocateBuffers();
 192 }
 193
 194 ChromotingStats* PepperVideoRenderer2D::GetStats() {
 195   DCHECK(CalledOnValidThread());
 196
 197   return software_video_renderer_->GetStats();
 198 }
 199
 200 protocol::VideoStub* PepperVideoRenderer2D::GetVideoStub() {
 201   DCHECK(CalledOnValidThread());
 202
 203   return software_video_renderer_->GetVideoStub();
 204 }
 205
 206 void PepperVideoRenderer2D::ApplyBuffer(const webrtc::DesktopSize& view_size,
 207                                         const webrtc::DesktopRect& clip_area,
 208                                         webrtc::DesktopFrame* buffer,
 209                                         const webrtc::DesktopRegion& region,
 210                                         const webrtc::DesktopRegion& shape) {
 211   DCHECK(CalledOnValidThread());
 212
 213   if (!frame_received_) {
 214     event_handler_->OnVideoFirstFrameReceived();
 215     frame_received_ = true;
 216   }
 217   // We cannot use the data in the buffer if its dimensions don't match the
 218   // current view size.
 219   // TODO(alexeypa): We could rescale and draw it (or even draw it without
 220   // rescaling) to reduce the perceived lag while we are waiting for
 221   // the properly scaled data.
 222   if (!view_size_.equals(view_size)) {
 223     FreeBuffer(buffer);
 224     AllocateBuffers();
 225   } else {
 226     FlushBuffer(clip_area, buffer, region);
 227     event_handler_->OnVideoShape(shape);
 228   }
 229 }
 230
 231 void PepperVideoRenderer2D::ReturnBuffer(webrtc::DesktopFrame* buffer) {
 232   DCHECK(CalledOnValidThread());
 233
 234   // Reuse the buffer if it is large enough, otherwise drop it on the floor
 235   // and allocate a new one.
 236   if (buffer->size().width() >= clip_area_.width() &&
 237       buffer->size().height() >= clip_area_.height()) {
 238     software_video_renderer_->DrawBuffer(buffer);
 239   } else {
 240     FreeBuffer(buffer);
 241     AllocateBuffers();
 242   }
 243 }
 244
 245 void PepperVideoRenderer2D::SetSourceSize(
 246     const webrtc::DesktopSize& source_size,
 247     const webrtc::DesktopVector& source_dpi) {
 248   DCHECK(CalledOnValidThread());
 249
 250   if (source_size_.equals(source_size) && source_dpi_.equals(source_dpi))
 251     return;
 252
 253   source_size_ = source_size;
 254   source_dpi_ = source_dpi;
 255
 256   // Notify JavaScript of the change in source size.
 257   event_handler_->OnVideoSize(source_size, source_dpi);
 258 }
 259
 260 FrameConsumer::PixelFormat PepperVideoRenderer2D::GetPixelFormat() {
 261   return FORMAT_BGRA;
 262 }
 263
 264 void PepperVideoRenderer2D::AllocateBuffers() {
 265   if (clip_area_.width() == 0 || clip_area_.height() == 0)
 266     return;
 267
 268   while (buffers_.size() < kMaxPendingBuffersCount) {
 269     // Create an image buffer of the required size, but don't zero it.
 270     pp::ImageData buffer_data(instance_, PP_IMAGEDATAFORMAT_BGRA_PREMUL,
 271                               pp::Size(clip_area_.width(), clip_area_.height()),
 272                               false);
 273     if (buffer_data.is_null()) {
 274       LOG(WARNING) << "Not enough memory for frame buffers.";
 275       break;
 276     }
 277
 278     webrtc::DesktopFrame* buffer = new PepperDesktopFrame(buffer_data);
 279     buffers_.push_back(buffer);
 280     software_video_renderer_->DrawBuffer(buffer);
 281   }
 282 }
 283
 284 void PepperVideoRenderer2D::FreeBuffer(webrtc::DesktopFrame* buffer) {
 285   DCHECK(std::find(buffers_.begin(), buffers_.end(), buffer) != buffers_.end());
 286
 287   buffers_.remove(buffer);
 288   delete buffer;
 289 }
 290
 291 void PepperVideoRenderer2D::FlushBuffer(const webrtc::DesktopRect& clip_area,
 292                                         webrtc::DesktopFrame* buffer,
 293                                         const webrtc::DesktopRegion& region) {
 294   // Defer drawing if the flush is already in progress.
 295   if (flush_pending_) {
 296     // |merge_buffer_| is guaranteed to be free here because we allocate only
 297     // two buffers simultaneously. If more buffers are allowed this code should
 298     // apply all pending changes to the screen.
 299     DCHECK(merge_buffer_ == nullptr);
 300
 301     merge_clip_area_ = clip_area;
 302     merge_buffer_ = buffer;
 303     merge_region_ = region;
 304     return;
 305   }
 306
 307   // Notify Pepper API about the updated areas and flush pixels to the screen.
 308   base::Time start_time = base::Time::Now();
 309
 310   for (webrtc::DesktopRegion::Iterator i(region); !i.IsAtEnd(); i.Advance()) {
 311     webrtc::DesktopRect rect = i.rect();
 312
 313     // Re-clip |region| with the current clipping area |clip_area_| because
 314     // the latter could change from the time the buffer was drawn.
 315     rect.IntersectWith(clip_area_);
 316     if (rect.is_empty())
 317       continue;
 318
 319     // Specify the rectangle coordinates relative to the clipping area.
 320     rect.Translate(-clip_area.left(), -clip_area.top());
 321
 322     // Pepper Graphics 2D has a strange and badly documented API that the
 323     // point here is the offset from the source rect. Why?
 324     graphics2d_.PaintImageData(
 325         static_cast<PepperDesktopFrame*>(buffer)->buffer(),
 326         pp::Point(clip_area.left(), clip_area.top()),
 327         pp::Rect(rect.left(), rect.top(), rect.width(), rect.height()));
 328   }
 329
 330   // Notify the producer that some parts of the region weren't painted because
 331   // the clipping area has changed already.
 332   if (!clip_area.equals(clip_area_)) {
 333     webrtc::DesktopRegion not_painted = region;
 334     not_painted.Subtract(clip_area_);
 335     if (!not_painted.is_empty()) {
 336       software_video_renderer_->InvalidateRegion(not_painted);
 337     }
 338   }
 339
 340   // Flush the updated areas to the screen.
 341   pp::CompletionCallback callback = callback_factory_.NewCallback(
 342       &PepperVideoRenderer2D::OnFlushDone, start_time, buffer);
 343   int error = graphics2d_.Flush(callback);
 344   CHECK(error == PP_OK_COMPLETIONPENDING);
 345   flush_pending_ = true;
 346
 347   // If Debug dirty region is enabled then emit it.
 348   if (debug_dirty_region_) {
 349     event_handler_->OnVideoFrameDirtyRegion(region);
 350   }
 351 }
 352
 353 void PepperVideoRenderer2D::OnFlushDone(int result,
 354                                         const base::Time& paint_start,
 355                                         webrtc::DesktopFrame* buffer) {
 356   DCHECK(CalledOnValidThread());
 357   DCHECK(flush_pending_);
 358
 359   software_video_renderer_->GetStats()->video_paint_ms()->Record(
 360       (base::Time::Now() - paint_start).InMilliseconds());
 361
 362   flush_pending_ = false;
 363   ReturnBuffer(buffer);
 364
 365   // If there is a buffer queued for rendering then render it now.
 366   if (merge_buffer_) {
 367     buffer = merge_buffer_;
 368     merge_buffer_ = nullptr;
 369     FlushBuffer(merge_clip_area_, buffer, merge_region_);
 370   }
 371 }
 372
 373 }  // namespace remoting