remoting/client/plugin/pepper_video_renderer_3d.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/plugin/pepper_video_renderer_3d.h"
   6
   7 #include <math.h>
   8
   9 #include "base/callback_helpers.h"
  10 #include "base/stl_util.h"
  11 #include "ppapi/c/pp_codecs.h"
  12 #include "ppapi/c/ppb_opengles2.h"
  13 #include "ppapi/c/ppb_video_decoder.h"
  14 #include "ppapi/cpp/instance.h"
  15 #include "ppapi/lib/gl/include/GLES2/gl2.h"
  16 #include "ppapi/lib/gl/include/GLES2/gl2ext.h"
  17 #include "remoting/proto/video.pb.h"
  18 #include "remoting/protocol/performance_tracker.h"
  19 #include "remoting/protocol/session_config.h"
  20
  21 namespace remoting {
  22
  23 // The implementation here requires this minimum number of pictures from the
  24 // video decoder interface to work.
  25 const uint32_t kMinimumPictureCount = 3;
  26
  27 class PepperVideoRenderer3D::PendingPacket {
  28  public:
  29   PendingPacket(scoped_ptr<VideoPacket> packet, const base::Closure& done)
  30       : packet_(packet.Pass()),
  31         done_runner_(done) {
  32   }
  33
  34   ~PendingPacket() {}
  35
  36   const VideoPacket* packet() const { return packet_.get(); }
  37
  38  private:
  39   scoped_ptr<VideoPacket> packet_;
  40   base::ScopedClosureRunner done_runner_;
  41 };
  42
  43
  44 class PepperVideoRenderer3D::Picture {
  45  public:
  46   Picture(pp::VideoDecoder* decoder, PP_VideoPicture picture)
  47       : decoder_(decoder), picture_(picture) {}
  48   ~Picture() { decoder_->RecyclePicture(picture_); }
  49
  50   const PP_VideoPicture& picture() { return picture_; }
  51
  52  private:
  53   pp::VideoDecoder* decoder_;
  54   PP_VideoPicture picture_;
  55 };
  56
  57 PepperVideoRenderer3D::FrameDecodeTimestamp::FrameDecodeTimestamp(
  58     uint32_t frame_id,
  59     base::TimeTicks decode_started_time)
  60     : frame_id(frame_id), decode_started_time(decode_started_time) {}
  61
  62 PepperVideoRenderer3D::PepperVideoRenderer3D() : callback_factory_(this) {}
  63
  64 PepperVideoRenderer3D::~PepperVideoRenderer3D() {
  65   if (shader_program_)
  66     gles2_if_->DeleteProgram(graphics_.pp_resource(), shader_program_);
  67
  68   STLDeleteElements(&pending_packets_);
  69 }
  70
  71 bool PepperVideoRenderer3D::Initialize(
  72     pp::Instance* instance,
  73     const ClientContext& context,
  74     EventHandler* event_handler,
  75     protocol::PerformanceTracker* perf_tracker) {
  76   DCHECK(event_handler);
  77   DCHECK(!event_handler_);
  78
  79   event_handler_ = event_handler;
  80   perf_tracker_ = perf_tracker;
  81
  82   const int32_t context_attributes[] = {
  83       PP_GRAPHICS3DATTRIB_ALPHA_SIZE,     8,
  84       PP_GRAPHICS3DATTRIB_BLUE_SIZE,      8,
  85       PP_GRAPHICS3DATTRIB_GREEN_SIZE,     8,
  86       PP_GRAPHICS3DATTRIB_RED_SIZE,       8,
  87       PP_GRAPHICS3DATTRIB_DEPTH_SIZE,     0,
  88       PP_GRAPHICS3DATTRIB_STENCIL_SIZE,   0,
  89       PP_GRAPHICS3DATTRIB_SAMPLES,        0,
  90       PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
  91       PP_GRAPHICS3DATTRIB_WIDTH,          640,
  92       PP_GRAPHICS3DATTRIB_HEIGHT,         480,
  93       PP_GRAPHICS3DATTRIB_NONE,
  94   };
  95   graphics_ = pp::Graphics3D(instance, context_attributes);
  96
  97   if (graphics_.is_null()) {
  98     LOG(WARNING) << "Graphics3D interface is not available.";
  99     return false;
 100   }
 101   if (!instance->BindGraphics(graphics_)) {
 102     LOG(WARNING) << "Failed to bind Graphics3D.";
 103     return false;
 104   }
 105
 106   // Fetch the GLES2 interface to use to render frames.
 107   gles2_if_ = static_cast<const PPB_OpenGLES2*>(
 108       pp::Module::Get()->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
 109   CHECK(gles2_if_);
 110
 111   video_decoder_ = pp::VideoDecoder(instance);
 112   if (video_decoder_.is_null()) {
 113     LOG(WARNING) << "VideoDecoder interface is not available.";
 114     return false;
 115   }
 116
 117   PP_Resource graphics_3d = graphics_.pp_resource();
 118
 119   gles2_if_->ClearColor(graphics_3d, 1, 0, 0, 1);
 120   gles2_if_->Clear(graphics_3d, GL_COLOR_BUFFER_BIT);
 121
 122   // Assign vertex positions and texture coordinates to buffers for use in
 123   // shader program.
 124   static const float kVertices[] = {
 125       -1, -1, -1, 1, 1, -1, 1, 1,  // Position coordinates.
 126       0,  1,  0,  0, 1, 1,  1, 0,  // Texture coordinates.
 127   };
 128
 129   GLuint buffer;
 130   gles2_if_->GenBuffers(graphics_3d, 1, &buffer);
 131   gles2_if_->BindBuffer(graphics_3d, GL_ARRAY_BUFFER, buffer);
 132   gles2_if_->BufferData(graphics_3d, GL_ARRAY_BUFFER, sizeof(kVertices),
 133                         kVertices, GL_STATIC_DRAW);
 134
 135   CheckGLError();
 136
 137   return true;
 138 }
 139
 140 void PepperVideoRenderer3D::OnViewChanged(const pp::View& view) {
 141   pp::Size size = view.GetRect().size();
 142   float scale = view.GetDeviceScale();
 143   view_size_.set(ceilf(size.width() * scale), ceilf(size.height() * scale));
 144   graphics_.ResizeBuffers(view_size_.width(), view_size_.height());
 145
 146   force_repaint_ = true;
 147   PaintIfNeeded();
 148 }
 149
 150 void PepperVideoRenderer3D::EnableDebugDirtyRegion(bool enable) {
 151   debug_dirty_region_ = enable;
 152 }
 153
 154 void PepperVideoRenderer3D::OnSessionConfig(
 155     const protocol::SessionConfig& config) {
 156   PP_VideoProfile video_profile = PP_VIDEOPROFILE_VP8_ANY;
 157   switch (config.video_config().codec) {
 158     case protocol::ChannelConfig::CODEC_VP8:
 159       video_profile = PP_VIDEOPROFILE_VP8_ANY;
 160       break;
 161     case protocol::ChannelConfig::CODEC_VP9:
 162       video_profile = PP_VIDEOPROFILE_VP9_ANY;
 163       break;
 164     default:
 165       NOTREACHED();
 166   }
 167
 168   bool supports_video_decoder_1_1 =
 169       pp::Module::Get()->GetBrowserInterface(
 170           PPB_VIDEODECODER_INTERFACE_1_1) != NULL;
 171   int32_t result = video_decoder_.Initialize(
 172       graphics_, video_profile, PP_HARDWAREACCELERATION_WITHFALLBACK,
 173       supports_video_decoder_1_1 ? kMinimumPictureCount : 0,
 174       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnInitialized));
 175   CHECK_EQ(result, PP_OK_COMPLETIONPENDING)
 176       << "video_decoder_.Initialize() returned " << result;
 177 }
 178
 179 protocol::VideoStub* PepperVideoRenderer3D::GetVideoStub() {
 180   return this;
 181 }
 182
 183 void PepperVideoRenderer3D::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
 184                                                const base::Closure& done) {
 185   base::ScopedClosureRunner done_runner(done);
 186
 187   perf_tracker_->RecordVideoPacketStats(*packet);
 188
 189   // Don't need to do anything if the packet is empty. Host sends empty video
 190   // packets when the screen is not changing.
 191   if (!packet->data().size())
 192     return;
 193
 194   bool resolution_changed = false;
 195
 196   if (packet->format().has_screen_width() &&
 197       packet->format().has_screen_height()) {
 198     webrtc::DesktopSize frame_size(packet->format().screen_width(),
 199                                    packet->format().screen_height());
 200     if (!frame_size_.equals(frame_size)) {
 201       frame_size_ = frame_size;
 202       resolution_changed = true;
 203     }
 204   }
 205
 206   if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) {
 207     webrtc::DesktopVector frame_dpi(packet->format().x_dpi(),
 208                                     packet->format().y_dpi());
 209     if (!frame_dpi_.equals(frame_dpi)) {
 210       frame_dpi_ = frame_dpi;
 211       resolution_changed = true;
 212     }
 213   }
 214
 215   if (resolution_changed)
 216     event_handler_->OnVideoSize(frame_size_, frame_dpi_);
 217
 218   // Process the frame shape, if supplied.
 219   if (packet->has_use_desktop_shape()) {
 220     if (packet->use_desktop_shape()) {
 221       scoped_ptr<webrtc::DesktopRegion> shape(new webrtc::DesktopRegion);
 222       for (int i = 0; i < packet->desktop_shape_rects_size(); ++i) {
 223         Rect remoting_rect = packet->desktop_shape_rects(i);
 224         shape->AddRect(webrtc::DesktopRect::MakeXYWH(
 225             remoting_rect.x(), remoting_rect.y(), remoting_rect.width(),
 226             remoting_rect.height()));
 227       }
 228       if (!frame_shape_ || !frame_shape_->Equals(*shape)) {
 229         frame_shape_ = shape.Pass();
 230         event_handler_->OnVideoShape(frame_shape_.get());
 231       }
 232     } else if (frame_shape_) {
 233       frame_shape_ = nullptr;
 234       event_handler_->OnVideoShape(nullptr);
 235     }
 236   }
 237
 238   // Report the dirty region, for debugging, if requested.
 239   if (debug_dirty_region_) {
 240     webrtc::DesktopRegion dirty_region;
 241     for (int i = 0; i < packet->dirty_rects_size(); ++i) {
 242       Rect remoting_rect = packet->dirty_rects(i);
 243       dirty_region.AddRect(webrtc::DesktopRect::MakeXYWH(
 244           remoting_rect.x(), remoting_rect.y(),
 245           remoting_rect.width(), remoting_rect.height()));
 246     }
 247     event_handler_->OnVideoFrameDirtyRegion(dirty_region);
 248   }
 249
 250   pending_packets_.push_back(
 251       new PendingPacket(packet.Pass(), done_runner.Release()));
 252   DecodeNextPacket();
 253 }
 254
 255 void PepperVideoRenderer3D::OnInitialized(int32_t result) {
 256   // Assume that VP8 and VP9 codecs are always supported by the browser.
 257   CHECK_EQ(result, PP_OK) << "VideoDecoder::Initialize() failed: " << result;
 258   initialization_finished_ = true;
 259
 260   // Start decoding in case a frame was received during decoder initialization.
 261   DecodeNextPacket();
 262 }
 263
 264 void PepperVideoRenderer3D::DecodeNextPacket() {
 265   if (!initialization_finished_ || decode_pending_ || pending_packets_.empty())
 266     return;
 267
 268   ++latest_frame_id_;
 269   frame_decode_timestamps_.push_back(
 270       FrameDecodeTimestamp(latest_frame_id_, base::TimeTicks::Now()));
 271
 272   const VideoPacket* packet = pending_packets_.front()->packet();
 273
 274   int32_t result = video_decoder_.Decode(
 275       latest_frame_id_, packet->data().size(), packet->data().data(),
 276       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnDecodeDone));
 277   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 278   decode_pending_ = true;
 279 }
 280
 281 void PepperVideoRenderer3D::OnDecodeDone(int32_t result) {
 282   DCHECK(decode_pending_);
 283   decode_pending_ = false;
 284
 285   if (result != PP_OK) {
 286     LOG(ERROR) << "VideoDecoder::Decode() returned " << result;
 287     event_handler_->OnVideoDecodeError();
 288     return;
 289   }
 290
 291   delete pending_packets_.front();
 292   pending_packets_.pop_front();
 293
 294   DecodeNextPacket();
 295   GetNextPicture();
 296 }
 297
 298 void PepperVideoRenderer3D::GetNextPicture() {
 299   if (get_picture_pending_)
 300     return;
 301
 302   int32_t result =
 303       video_decoder_.GetPicture(callback_factory_.NewCallbackWithOutput(
 304           &PepperVideoRenderer3D::OnPictureReady));
 305   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 306   get_picture_pending_ = true;
 307 }
 308
 309 void PepperVideoRenderer3D::OnPictureReady(int32_t result,
 310                                            PP_VideoPicture picture) {
 311   DCHECK(get_picture_pending_);
 312   get_picture_pending_ = false;
 313
 314   if (result != PP_OK) {
 315     LOG(ERROR) << "VideoDecoder::GetPicture() returned " << result;
 316     event_handler_->OnVideoDecodeError();
 317     return;
 318   }
 319
 320   CHECK(!frame_decode_timestamps_.empty());
 321   const FrameDecodeTimestamp& frame_timer = frame_decode_timestamps_.front();
 322
 323   if (picture.decode_id != frame_timer.frame_id) {
 324     LOG(ERROR)
 325         << "Received a video packet that didn't contain a complete frame.";
 326     event_handler_->OnVideoDecodeError();
 327     return;
 328   }
 329
 330   base::TimeDelta decode_time =
 331       base::TimeTicks::Now() - frame_timer.decode_started_time;
 332   perf_tracker_->RecordDecodeTime(decode_time.InMilliseconds());
 333
 334   frame_decode_timestamps_.pop_front();
 335
 336   next_picture_.reset(new Picture(&video_decoder_, picture));
 337
 338   PaintIfNeeded();
 339   GetNextPicture();
 340 }
 341
 342 void PepperVideoRenderer3D::PaintIfNeeded() {
 343   bool need_repaint = next_picture_ || (force_repaint_ && current_picture_);
 344   if (paint_pending_ || !need_repaint)
 345     return;
 346
 347   if (next_picture_)
 348     current_picture_ = next_picture_.Pass();
 349
 350   force_repaint_ = false;
 351   latest_paint_started_time_ = base::TimeTicks::Now();
 352
 353   const PP_VideoPicture& picture = current_picture_->picture();
 354   PP_Resource graphics_3d = graphics_.pp_resource();
 355
 356   EnsureProgramForTexture(picture.texture_target);
 357
 358   gles2_if_->UseProgram(graphics_3d, shader_program_);
 359
 360   // Calculate v_scale passed to the vertex shader.
 361   double scale_x = picture.visible_rect.size.width;
 362   double scale_y = picture.visible_rect.size.height;
 363   if (picture.texture_target != GL_TEXTURE_RECTANGLE_ARB) {
 364     scale_x /= picture.texture_size.width;
 365     scale_y /= picture.texture_size.height;
 366   }
 367   gles2_if_->Uniform2f(graphics_3d, shader_texcoord_scale_location_,
 368                        scale_x, scale_y);
 369
 370   // Set viewport position & dimensions.
 371   gles2_if_->Viewport(graphics_3d, 0, 0, view_size_.width(),
 372                       view_size_.height());
 373
 374   // Select the texture unit GL_TEXTURE0.
 375   gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
 376
 377   // Select the texture.
 378   gles2_if_->BindTexture(graphics_3d, picture.texture_target,
 379                          picture.texture_id);
 380
 381   // Select linear filter in case the texture needs to be scaled.
 382   gles2_if_->TexParameteri(graphics_3d, picture.texture_target,
 383                            GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 384
 385   // When view dimensions are a multiple of the frame size then use
 386   // nearest-neighbor scaling to achieve crisper image. Linear filter is used in
 387   // all other cases.
 388   GLint mag_filter = GL_LINEAR;
 389   if (view_size_.width() % picture.visible_rect.size.width == 0 &&
 390       view_size_.height() % picture.visible_rect.size.height == 0) {
 391     mag_filter = GL_NEAREST;
 392   }
 393   gles2_if_->TexParameteri(graphics_3d, picture.texture_target,
 394                            GL_TEXTURE_MAG_FILTER, mag_filter);
 395
 396   // Render texture by drawing a triangle strip with 4 vertices.
 397   gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
 398
 399   CheckGLError();
 400
 401   // Request PPAPI display the queued texture.
 402   int32_t result = graphics_.SwapBuffers(
 403       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnPaintDone));
 404   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 405   paint_pending_ = true;
 406 }
 407
 408 void PepperVideoRenderer3D::OnPaintDone(int32_t result) {
 409   CHECK_EQ(result, PP_OK) << "Graphics3D::SwapBuffers() failed";
 410
 411   paint_pending_ = false;
 412   base::TimeDelta paint_time =
 413       base::TimeTicks::Now() - latest_paint_started_time_;
 414   perf_tracker_->RecordPaintTime(paint_time.InMilliseconds());
 415   PaintIfNeeded();
 416 }
 417
 418 void PepperVideoRenderer3D::EnsureProgramForTexture(uint32_t texture_target) {
 419   static const char kVertexShader[] =
 420       "varying vec2 v_texCoord;            \n"
 421       "attribute vec4 a_position;          \n"
 422       "attribute vec2 a_texCoord;          \n"
 423       "uniform vec2 v_scale;               \n"
 424       "void main()                         \n"
 425       "{                                   \n"
 426       "    v_texCoord = v_scale * a_texCoord; \n"
 427       "    gl_Position = a_position;       \n"
 428       "}";
 429
 430   static const char kFragmentShader2D[] =
 431       "precision mediump float;            \n"
 432       "varying vec2 v_texCoord;            \n"
 433       "uniform sampler2D s_texture;        \n"
 434       "void main()                         \n"
 435       "{"
 436       "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
 437       "}";
 438
 439   static const char kFragmentShaderRectangle[] =
 440       "#extension GL_ARB_texture_rectangle : require\n"
 441       "precision mediump float;            \n"
 442       "varying vec2 v_texCoord;            \n"
 443       "uniform sampler2DRect s_texture;    \n"
 444       "void main()                         \n"
 445       "{"
 446       "    gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
 447       "}";
 448
 449   static const char kFragmentShaderExternal[] =
 450       "#extension GL_OES_EGL_image_external : require\n"
 451       "precision mediump float;            \n"
 452       "varying vec2 v_texCoord;            \n"
 453       "uniform samplerExternalOES s_texture; \n"
 454       "void main()                         \n"
 455       "{"
 456       "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
 457       "}";
 458
 459   // Initialize shader program only if texture type has changed.
 460   if (current_shader_program_texture_target_ != texture_target) {
 461     current_shader_program_texture_target_ = texture_target;
 462
 463     if (texture_target == GL_TEXTURE_2D) {
 464       CreateProgram(kVertexShader, kFragmentShader2D);
 465     } else if (texture_target == GL_TEXTURE_RECTANGLE_ARB) {
 466       CreateProgram(kVertexShader, kFragmentShaderRectangle);
 467     } else if (texture_target == GL_TEXTURE_EXTERNAL_OES) {
 468       CreateProgram(kVertexShader, kFragmentShaderExternal);
 469     } else {
 470       LOG(FATAL) << "Unknown texture target: " << texture_target;
 471     }
 472   }
 473 }
 474
 475 void PepperVideoRenderer3D::CreateProgram(const char* vertex_shader,
 476                                           const char* fragment_shader) {
 477   PP_Resource graphics_3d = graphics_.pp_resource();
 478   if (shader_program_)
 479     gles2_if_->DeleteProgram(graphics_3d, shader_program_);
 480
 481   // Create shader program.
 482   shader_program_ = gles2_if_->CreateProgram(graphics_3d);
 483   CreateShaderProgram(GL_VERTEX_SHADER, vertex_shader);
 484   CreateShaderProgram(GL_FRAGMENT_SHADER, fragment_shader);
 485   gles2_if_->LinkProgram(graphics_3d, shader_program_);
 486   gles2_if_->UseProgram(graphics_3d, shader_program_);
 487   gles2_if_->Uniform1i(
 488       graphics_3d,
 489       gles2_if_->GetUniformLocation(graphics_3d, shader_program_, "s_texture"),
 490       0);
 491   CheckGLError();
 492
 493   shader_texcoord_scale_location_ = gles2_if_->GetUniformLocation(
 494       graphics_3d, shader_program_, "v_scale");
 495
 496   GLint pos_location = gles2_if_->GetAttribLocation(
 497       graphics_3d, shader_program_, "a_position");
 498   GLint tc_location = gles2_if_->GetAttribLocation(
 499       graphics_3d, shader_program_, "a_texCoord");
 500   CheckGLError();
 501
 502   // Construct the vertex array for DrawArrays(), using the buffer created in
 503   // Initialize().
 504   gles2_if_->EnableVertexAttribArray(graphics_3d, pos_location);
 505   gles2_if_->VertexAttribPointer(graphics_3d, pos_location, 2, GL_FLOAT,
 506                                  GL_FALSE, 0, 0);
 507   gles2_if_->EnableVertexAttribArray(graphics_3d, tc_location);
 508   gles2_if_->VertexAttribPointer(
 509       graphics_3d, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 510       static_cast<float*>(0) + 8);  // Skip position coordinates.
 511
 512   gles2_if_->UseProgram(graphics_3d, 0);
 513
 514   CheckGLError();
 515 }
 516
 517 void PepperVideoRenderer3D::CreateShaderProgram(int type, const char* source) {
 518   int size = strlen(source);
 519   GLuint shader = gles2_if_->CreateShader(graphics_.pp_resource(), type);
 520   gles2_if_->ShaderSource(graphics_.pp_resource(), shader, 1, &source, &size);
 521   gles2_if_->CompileShader(graphics_.pp_resource(), shader);
 522   gles2_if_->AttachShader(graphics_.pp_resource(), shader_program_, shader);
 523   gles2_if_->DeleteShader(graphics_.pp_resource(), shader);
 524 }
 525
 526 void PepperVideoRenderer3D::CheckGLError() {
 527   GLenum error = gles2_if_->GetError(graphics_.pp_resource());
 528   CHECK_EQ(error, static_cast<GLenum>(GL_NO_ERROR)) << "GL error: " << error;
 529 }
 530
 531 }  // namespace remoting