remoting/client/plugin/pepper_video_renderer_3d.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/plugin/pepper_video_renderer_3d.h"
   6
   7 #include <math.h>
   8
   9 #include "base/callback_helpers.h"
  10 #include "base/stl_util.h"
  11 #include "ppapi/c/pp_codecs.h"
  12 #include "ppapi/c/ppb_opengles2.h"
  13 #include "ppapi/cpp/instance.h"
  14 #include "ppapi/lib/gl/include/GLES2/gl2.h"
  15 #include "ppapi/lib/gl/include/GLES2/gl2ext.h"
  16 #include "remoting/proto/video.pb.h"
  17 #include "remoting/protocol/session_config.h"
  18
  19 namespace remoting {
  20
  21 class PepperVideoRenderer3D::PendingPacket {
  22  public:
  23   PendingPacket(scoped_ptr<VideoPacket> packet, const base::Closure& done)
  24       : packet_(packet.Pass()),
  25         done_runner_(done) {
  26   }
  27
  28   ~PendingPacket() {}
  29
  30   const VideoPacket* packet() const { return packet_.get(); }
  31
  32  private:
  33   scoped_ptr<VideoPacket> packet_;
  34   base::ScopedClosureRunner done_runner_;
  35 };
  36
  37
  38 class PepperVideoRenderer3D::Picture {
  39  public:
  40   Picture(pp::VideoDecoder* decoder, PP_VideoPicture picture)
  41       : decoder_(decoder), picture_(picture) {}
  42   ~Picture() { decoder_->RecyclePicture(picture_); }
  43
  44   const PP_VideoPicture& picture() { return picture_; }
  45
  46  private:
  47   pp::VideoDecoder* decoder_;
  48   PP_VideoPicture picture_;
  49 };
  50
  51
  52 PepperVideoRenderer3D::FrameDecodeTimestamp::FrameDecodeTimestamp(
  53     uint32_t frame_id,
  54     base::TimeTicks decode_started_time)
  55     : frame_id(frame_id), decode_started_time(decode_started_time) {
  56 }
  57
  58 PepperVideoRenderer3D::PepperVideoRenderer3D()
  59     : event_handler_(nullptr),
  60       latest_input_event_timestamp_(0),
  61       initialization_finished_(false),
  62       decode_pending_(false),
  63       get_picture_pending_(false),
  64       paint_pending_(false),
  65       latest_frame_id_(0),
  66       force_repaint_(false),
  67       current_shader_program_texture_target_(0),
  68       shader_program_(0),
  69       shader_texcoord_scale_location_(0),
  70       callback_factory_(this) {
  71 }
  72
  73 PepperVideoRenderer3D::~PepperVideoRenderer3D() {
  74   if (shader_program_)
  75     gles2_if_->DeleteProgram(graphics_.pp_resource(), shader_program_);
  76
  77   STLDeleteElements(&pending_packets_);
  78 }
  79
  80 bool PepperVideoRenderer3D::Initialize(pp::Instance* instance,
  81                                        const ClientContext& context,
  82                                        EventHandler* event_handler) {
  83   DCHECK(event_handler);
  84   DCHECK(!event_handler_);
  85
  86   event_handler_ = event_handler;
  87
  88   const int32_t context_attributes[] = {
  89       PP_GRAPHICS3DATTRIB_ALPHA_SIZE,     8,
  90       PP_GRAPHICS3DATTRIB_BLUE_SIZE,      8,
  91       PP_GRAPHICS3DATTRIB_GREEN_SIZE,     8,
  92       PP_GRAPHICS3DATTRIB_RED_SIZE,       8,
  93       PP_GRAPHICS3DATTRIB_DEPTH_SIZE,     0,
  94       PP_GRAPHICS3DATTRIB_STENCIL_SIZE,   0,
  95       PP_GRAPHICS3DATTRIB_SAMPLES,        0,
  96       PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
  97       PP_GRAPHICS3DATTRIB_WIDTH,          640,
  98       PP_GRAPHICS3DATTRIB_HEIGHT,         480,
  99       PP_GRAPHICS3DATTRIB_NONE,
 100   };
 101   graphics_ = pp::Graphics3D(instance, context_attributes);
 102
 103   if (graphics_.is_null()) {
 104     LOG(WARNING) << "Graphics3D interface is not available.";
 105     return false;
 106   }
 107   if (!instance->BindGraphics(graphics_)) {
 108     LOG(WARNING) << "Failed to bind Graphics3D.";
 109     return false;
 110   }
 111
 112   // Fetch the GLES2 interface to use to render frames.
 113   gles2_if_ = static_cast<const PPB_OpenGLES2*>(
 114       pp::Module::Get()->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
 115   CHECK(gles2_if_);
 116
 117   video_decoder_ = pp::VideoDecoder(instance);
 118   if (video_decoder_.is_null()) {
 119     LOG(WARNING) << "VideoDecoder interface is not available.";
 120     return false;
 121   }
 122
 123   PP_Resource graphics_3d = graphics_.pp_resource();
 124
 125   gles2_if_->ClearColor(graphics_3d, 1, 0, 0, 1);
 126   gles2_if_->Clear(graphics_3d, GL_COLOR_BUFFER_BIT);
 127
 128   // Assign vertex positions and texture coordinates to buffers for use in
 129   // shader program.
 130   static const float kVertices[] = {
 131       -1, -1, -1, 1, 1, -1, 1, 1,  // Position coordinates.
 132       0,  1,  0,  0, 1, 1,  1, 0,  // Texture coordinates.
 133   };
 134
 135   GLuint buffer;
 136   gles2_if_->GenBuffers(graphics_3d, 1, &buffer);
 137   gles2_if_->BindBuffer(graphics_3d, GL_ARRAY_BUFFER, buffer);
 138   gles2_if_->BufferData(graphics_3d, GL_ARRAY_BUFFER, sizeof(kVertices),
 139                         kVertices, GL_STATIC_DRAW);
 140
 141   CheckGLError();
 142
 143   return true;
 144 }
 145
 146 void PepperVideoRenderer3D::OnViewChanged(const pp::View& view) {
 147   pp::Size size = view.GetRect().size();
 148   float scale = view.GetDeviceScale();
 149   view_size_.set(ceilf(size.width() * scale), ceilf(size.height() * scale));
 150   graphics_.ResizeBuffers(view_size_.width(), view_size_.height());
 151
 152   force_repaint_ = true;
 153   PaintIfNeeded();
 154 }
 155
 156 void PepperVideoRenderer3D::EnableDebugDirtyRegion(bool enable) {
 157   debug_dirty_region_ = enable;
 158 }
 159
 160 void PepperVideoRenderer3D::OnSessionConfig(
 161     const protocol::SessionConfig& config) {
 162   PP_VideoProfile video_profile = PP_VIDEOPROFILE_VP8_ANY;
 163   switch (config.video_config().codec) {
 164     case protocol::ChannelConfig::CODEC_VP8:
 165       video_profile = PP_VIDEOPROFILE_VP8_ANY;
 166       break;
 167     case protocol::ChannelConfig::CODEC_VP9:
 168       video_profile = PP_VIDEOPROFILE_VP9_ANY;
 169       break;
 170     default:
 171       NOTREACHED();
 172   }
 173   int32_t result = video_decoder_.Initialize(
 174       graphics_, video_profile, PP_HARDWAREACCELERATION_WITHFALLBACK,
 175       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnInitialized));
 176   CHECK_EQ(result, PP_OK_COMPLETIONPENDING)
 177       << "video_decoder_.Initialize() returned " << result;
 178 }
 179
 180 ChromotingStats* PepperVideoRenderer3D::GetStats() {
 181   return &stats_;
 182 }
 183
 184 protocol::VideoStub* PepperVideoRenderer3D::GetVideoStub() {
 185   return this;
 186 }
 187
 188 void PepperVideoRenderer3D::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
 189                                                const base::Closure& done) {
 190   base::ScopedClosureRunner done_runner(done);
 191
 192   // Don't need to do anything if the packet is empty. Host sends empty video
 193   // packets when the screen is not changing.
 194   if (!packet->data().size())
 195     return;
 196
 197   // Update statistics.
 198   stats_.video_frame_rate()->Record(1);
 199   stats_.video_bandwidth()->Record(packet->data().size());
 200   if (packet->has_capture_time_ms())
 201     stats_.video_capture_ms()->Record(packet->capture_time_ms());
 202   if (packet->has_encode_time_ms())
 203     stats_.video_encode_ms()->Record(packet->encode_time_ms());
 204   if (packet->has_latest_event_timestamp() &&
 205       packet->latest_event_timestamp() > latest_input_event_timestamp_) {
 206     latest_input_event_timestamp_ = packet->latest_event_timestamp();
 207     base::TimeDelta round_trip_latency =
 208         base::Time::Now() -
 209         base::Time::FromInternalValue(packet->latest_event_timestamp());
 210     stats_.round_trip_ms()->Record(round_trip_latency.InMilliseconds());
 211   }
 212
 213   bool resolution_changed = false;
 214
 215   if (packet->format().has_screen_width() &&
 216       packet->format().has_screen_height()) {
 217     webrtc::DesktopSize frame_size(packet->format().screen_width(),
 218                                    packet->format().screen_height());
 219     if (!frame_size_.equals(frame_size)) {
 220       frame_size_ = frame_size;
 221       resolution_changed = true;
 222     }
 223   }
 224
 225   if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) {
 226     webrtc::DesktopVector frame_dpi(packet->format().x_dpi(),
 227                                     packet->format().y_dpi());
 228     if (!frame_dpi_.equals(frame_dpi)) {
 229       frame_dpi_ = frame_dpi;
 230       resolution_changed = true;
 231     }
 232   }
 233
 234   if (resolution_changed)
 235     event_handler_->OnVideoSize(frame_size_, frame_dpi_);
 236
 237   // Update the desktop shape region.
 238   webrtc::DesktopRegion desktop_shape;
 239   if (packet->has_use_desktop_shape()) {
 240     for (int i = 0; i < packet->desktop_shape_rects_size(); ++i) {
 241       Rect remoting_rect = packet->desktop_shape_rects(i);
 242       desktop_shape.AddRect(webrtc::DesktopRect::MakeXYWH(
 243           remoting_rect.x(), remoting_rect.y(),
 244           remoting_rect.width(), remoting_rect.height()));
 245     }
 246   } else {
 247     // Fallback for the case when the host didn't include the desktop shape.
 248     desktop_shape =
 249         webrtc::DesktopRegion(webrtc::DesktopRect::MakeSize(frame_size_));
 250   }
 251
 252   if (!desktop_shape_.Equals(desktop_shape)) {
 253     desktop_shape_.Swap(&desktop_shape);
 254     event_handler_->OnVideoShape(desktop_shape_);
 255   }
 256
 257   // Report the dirty region, for debugging, if requested.
 258   if (debug_dirty_region_) {
 259     webrtc::DesktopRegion dirty_region;
 260     for (int i = 0; i < packet->dirty_rects_size(); ++i) {
 261       Rect remoting_rect = packet->dirty_rects(i);
 262       dirty_region.AddRect(webrtc::DesktopRect::MakeXYWH(
 263           remoting_rect.x(), remoting_rect.y(),
 264           remoting_rect.width(), remoting_rect.height()));
 265     }
 266     event_handler_->OnVideoFrameDirtyRegion(dirty_region);
 267   }
 268
 269   pending_packets_.push_back(
 270       new PendingPacket(packet.Pass(), done_runner.Release()));
 271   DecodeNextPacket();
 272 }
 273
 274 void PepperVideoRenderer3D::OnInitialized(int32_t result) {
 275   // Assume that VP8 and VP9 codecs are always supported by the browser.
 276   CHECK_EQ(result, PP_OK) << "VideoDecoder::Initialize() failed: " << result;
 277   initialization_finished_ = true;
 278
 279   // Start decoding in case a frame was received during decoder initialization.
 280   DecodeNextPacket();
 281 }
 282
 283 void PepperVideoRenderer3D::DecodeNextPacket() {
 284   if (!initialization_finished_ || decode_pending_ || pending_packets_.empty())
 285     return;
 286
 287   ++latest_frame_id_;
 288   frame_decode_timestamps_.push_back(
 289       FrameDecodeTimestamp(latest_frame_id_, base::TimeTicks::Now()));
 290
 291   const VideoPacket* packet = pending_packets_.front()->packet();
 292
 293   int32_t result = video_decoder_.Decode(
 294       latest_frame_id_, packet->data().size(), packet->data().data(),
 295       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnDecodeDone));
 296   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 297   decode_pending_ = true;
 298 }
 299
 300 void PepperVideoRenderer3D::OnDecodeDone(int32_t result) {
 301   DCHECK(decode_pending_);
 302   decode_pending_ = false;
 303
 304   if (result != PP_OK) {
 305     LOG(ERROR) << "VideoDecoder::Decode() returned " << result;
 306     event_handler_->OnVideoDecodeError();
 307     return;
 308   }
 309
 310   delete pending_packets_.front();
 311   pending_packets_.pop_front();
 312
 313   DecodeNextPacket();
 314   GetNextPicture();
 315 }
 316
 317 void PepperVideoRenderer3D::GetNextPicture() {
 318   if (get_picture_pending_)
 319     return;
 320
 321   int32_t result =
 322       video_decoder_.GetPicture(callback_factory_.NewCallbackWithOutput(
 323           &PepperVideoRenderer3D::OnPictureReady));
 324   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 325   get_picture_pending_ = true;
 326 }
 327
 328 void PepperVideoRenderer3D::OnPictureReady(int32_t result,
 329                                            PP_VideoPicture picture) {
 330   DCHECK(get_picture_pending_);
 331   get_picture_pending_ = false;
 332
 333   if (result != PP_OK) {
 334     LOG(ERROR) << "VideoDecoder::GetPicture() returned " << result;
 335     event_handler_->OnVideoDecodeError();
 336     return;
 337   }
 338
 339   CHECK(!frame_decode_timestamps_.empty());
 340   const FrameDecodeTimestamp& frame_timer = frame_decode_timestamps_.front();
 341
 342   if (picture.decode_id != frame_timer.frame_id) {
 343     LOG(ERROR)
 344         << "Received a video packet that didn't contain a complete frame.";
 345     event_handler_->OnVideoDecodeError();
 346     return;
 347   }
 348
 349   base::TimeDelta decode_time =
 350       base::TimeTicks::Now() - frame_timer.decode_started_time;
 351   stats_.video_decode_ms()->Record(decode_time.InMilliseconds());
 352   frame_decode_timestamps_.pop_front();
 353
 354   next_picture_.reset(new Picture(&video_decoder_, picture));
 355
 356   PaintIfNeeded();
 357   GetNextPicture();
 358 }
 359
 360 void PepperVideoRenderer3D::PaintIfNeeded() {
 361   bool need_repaint = next_picture_ || (force_repaint_ && current_picture_);
 362   if (paint_pending_ || !need_repaint)
 363     return;
 364
 365   if (next_picture_)
 366     current_picture_ = next_picture_.Pass();
 367
 368   force_repaint_ = false;
 369   latest_paint_started_time_ = base::TimeTicks::Now();
 370
 371   const PP_VideoPicture& picture = current_picture_->picture();
 372   PP_Resource graphics_3d = graphics_.pp_resource();
 373
 374   EnsureProgramForTexture(picture.texture_target);
 375
 376   gles2_if_->UseProgram(graphics_3d, shader_program_);
 377
 378   // Calculate v_scale passed to the vertex shader.
 379   double scale_x = picture.visible_rect.size.width;
 380   double scale_y = picture.visible_rect.size.height;
 381   if (picture.texture_target != GL_TEXTURE_RECTANGLE_ARB) {
 382     scale_x /= picture.texture_size.width;
 383     scale_y /= picture.texture_size.height;
 384   }
 385   gles2_if_->Uniform2f(graphics_3d, shader_texcoord_scale_location_,
 386                        scale_x, scale_y);
 387
 388   // Set viewport position & dimensions.
 389   gles2_if_->Viewport(graphics_3d, 0, 0, view_size_.width(),
 390                       view_size_.height());
 391
 392   // Select the texture unit GL_TEXTURE0.
 393   gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
 394
 395   // Select the texture.
 396   gles2_if_->BindTexture(graphics_3d, picture.texture_target,
 397                          picture.texture_id);
 398
 399   // Select linear filter in case the texture needs to be scaled.
 400   gles2_if_->TexParameteri(graphics_3d, picture.texture_target,
 401                            GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 402
 403   // Render texture by drawing a triangle strip with 4 vertices.
 404   gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
 405
 406   CheckGLError();
 407
 408   // Request PPAPI display the queued texture.
 409   int32_t result = graphics_.SwapBuffers(
 410       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnPaintDone));
 411   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 412   paint_pending_ = true;
 413 }
 414
 415 void PepperVideoRenderer3D::OnPaintDone(int32_t result) {
 416   CHECK_EQ(result, PP_OK) << "Graphics3D::SwapBuffers() failed";
 417
 418   paint_pending_ = false;
 419   base::TimeDelta paint_time =
 420       base::TimeTicks::Now() - latest_paint_started_time_;
 421   stats_.video_paint_ms()->Record(paint_time.InMilliseconds());
 422
 423   PaintIfNeeded();
 424 }
 425
 426 void PepperVideoRenderer3D::EnsureProgramForTexture(uint32_t texture_target) {
 427   static const char kVertexShader[] =
 428       "varying vec2 v_texCoord;            \n"
 429       "attribute vec4 a_position;          \n"
 430       "attribute vec2 a_texCoord;          \n"
 431       "uniform vec2 v_scale;               \n"
 432       "void main()                         \n"
 433       "{                                   \n"
 434       "    v_texCoord = v_scale * a_texCoord; \n"
 435       "    gl_Position = a_position;       \n"
 436       "}";
 437
 438   static const char kFragmentShader2D[] =
 439       "precision mediump float;            \n"
 440       "varying vec2 v_texCoord;            \n"
 441       "uniform sampler2D s_texture;        \n"
 442       "void main()                         \n"
 443       "{"
 444       "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
 445       "}";
 446
 447   static const char kFragmentShaderRectangle[] =
 448       "#extension GL_ARB_texture_rectangle : require\n"
 449       "precision mediump float;            \n"
 450       "varying vec2 v_texCoord;            \n"
 451       "uniform sampler2DRect s_texture;    \n"
 452       "void main()                         \n"
 453       "{"
 454       "    gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
 455       "}";
 456
 457   static const char kFragmentShaderExternal[] =
 458       "#extension GL_OES_EGL_image_external : require\n"
 459       "precision mediump float;            \n"
 460       "varying vec2 v_texCoord;            \n"
 461       "uniform samplerExternalOES s_texture; \n"
 462       "void main()                         \n"
 463       "{"
 464       "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
 465       "}";
 466
 467   // Initialize shader program only if texture type has changed.
 468   if (current_shader_program_texture_target_ != texture_target) {
 469     current_shader_program_texture_target_ = texture_target;
 470
 471     if (texture_target == GL_TEXTURE_2D) {
 472       CreateProgram(kVertexShader, kFragmentShader2D);
 473     } else if (texture_target == GL_TEXTURE_RECTANGLE_ARB) {
 474       CreateProgram(kVertexShader, kFragmentShaderRectangle);
 475     } else if (texture_target == GL_TEXTURE_EXTERNAL_OES) {
 476       CreateProgram(kVertexShader, kFragmentShaderExternal);
 477     } else {
 478       LOG(FATAL) << "Unknown texture target: " << texture_target;
 479     }
 480   }
 481 }
 482
 483 void PepperVideoRenderer3D::CreateProgram(const char* vertex_shader,
 484                                           const char* fragment_shader) {
 485   PP_Resource graphics_3d = graphics_.pp_resource();
 486   if (shader_program_)
 487     gles2_if_->DeleteProgram(graphics_3d, shader_program_);
 488
 489   // Create shader program.
 490   shader_program_ = gles2_if_->CreateProgram(graphics_3d);
 491   CreateShaderProgram(GL_VERTEX_SHADER, vertex_shader);
 492   CreateShaderProgram(GL_FRAGMENT_SHADER, fragment_shader);
 493   gles2_if_->LinkProgram(graphics_3d, shader_program_);
 494   gles2_if_->UseProgram(graphics_3d, shader_program_);
 495   gles2_if_->Uniform1i(
 496       graphics_3d,
 497       gles2_if_->GetUniformLocation(graphics_3d, shader_program_, "s_texture"),
 498       0);
 499   CheckGLError();
 500
 501   shader_texcoord_scale_location_ = gles2_if_->GetUniformLocation(
 502       graphics_3d, shader_program_, "v_scale");
 503
 504   GLint pos_location = gles2_if_->GetAttribLocation(
 505       graphics_3d, shader_program_, "a_position");
 506   GLint tc_location = gles2_if_->GetAttribLocation(
 507       graphics_3d, shader_program_, "a_texCoord");
 508   CheckGLError();
 509
 510   // Construct the vertex array for DrawArrays(), using the buffer created in
 511   // Initialize().
 512   gles2_if_->EnableVertexAttribArray(graphics_3d, pos_location);
 513   gles2_if_->VertexAttribPointer(graphics_3d, pos_location, 2, GL_FLOAT,
 514                                  GL_FALSE, 0, 0);
 515   gles2_if_->EnableVertexAttribArray(graphics_3d, tc_location);
 516   gles2_if_->VertexAttribPointer(
 517       graphics_3d, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 518       static_cast<float*>(0) + 8);  // Skip position coordinates.
 519
 520   gles2_if_->UseProgram(graphics_3d, 0);
 521
 522   CheckGLError();
 523 }
 524
 525 void PepperVideoRenderer3D::CreateShaderProgram(int type, const char* source) {
 526   int size = strlen(source);
 527   GLuint shader = gles2_if_->CreateShader(graphics_.pp_resource(), type);
 528   gles2_if_->ShaderSource(graphics_.pp_resource(), shader, 1, &source, &size);
 529   gles2_if_->CompileShader(graphics_.pp_resource(), shader);
 530   gles2_if_->AttachShader(graphics_.pp_resource(), shader_program_, shader);
 531   gles2_if_->DeleteShader(graphics_.pp_resource(), shader);
 532 }
 533
 534 void PepperVideoRenderer3D::CheckGLError() {
 535   GLenum error = gles2_if_->GetError(graphics_.pp_resource());
 536   CHECK_EQ(error, static_cast<GLenum>(GL_NO_ERROR)) << "GL error: " << error;
 537 }
 538
 539 }  // namespace remoting