remoting/client/plugin/pepper_video_renderer_3d.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/client/plugin/pepper_video_renderer_3d.h"
   6
   7 #include <math.h>
   8
   9 #include "base/callback_helpers.h"
  10 #include "base/stl_util.h"
  11 #include "ppapi/c/pp_codecs.h"
  12 #include "ppapi/c/ppb_opengles2.h"
  13 #include "ppapi/c/ppb_video_decoder.h"
  14 #include "ppapi/cpp/instance.h"
  15 #include "ppapi/lib/gl/include/GLES2/gl2.h"
  16 #include "ppapi/lib/gl/include/GLES2/gl2ext.h"
  17 #include "remoting/client/chromoting_stats.h"
  18 #include "remoting/proto/video.pb.h"
  19 #include "remoting/protocol/session_config.h"
  20
  21 namespace remoting {
  22
  23 // The implementation here requires this minimum number of pictures from the
  24 // video decoder interface to work.
  25 const uint32_t kMinimumPictureCount = 3;
  26
  27 class PepperVideoRenderer3D::PendingPacket {
  28  public:
  29   PendingPacket(scoped_ptr<VideoPacket> packet, const base::Closure& done)
  30       : packet_(packet.Pass()),
  31         done_runner_(done) {
  32   }
  33
  34   ~PendingPacket() {}
  35
  36   const VideoPacket* packet() const { return packet_.get(); }
  37
  38  private:
  39   scoped_ptr<VideoPacket> packet_;
  40   base::ScopedClosureRunner done_runner_;
  41 };
  42
  43
  44 class PepperVideoRenderer3D::Picture {
  45  public:
  46   Picture(pp::VideoDecoder* decoder, PP_VideoPicture picture)
  47       : decoder_(decoder), picture_(picture) {}
  48   ~Picture() { decoder_->RecyclePicture(picture_); }
  49
  50   const PP_VideoPicture& picture() { return picture_; }
  51
  52  private:
  53   pp::VideoDecoder* decoder_;
  54   PP_VideoPicture picture_;
  55 };
  56
  57
  58 PepperVideoRenderer3D::FrameDecodeTimestamp::FrameDecodeTimestamp(
  59     uint32_t frame_id,
  60     base::TimeTicks decode_started_time)
  61     : frame_id(frame_id), decode_started_time(decode_started_time) {
  62 }
  63
  64 PepperVideoRenderer3D::PepperVideoRenderer3D()
  65     : event_handler_(nullptr),
  66       initialization_finished_(false),
  67       decode_pending_(false),
  68       get_picture_pending_(false),
  69       paint_pending_(false),
  70       latest_frame_id_(0),
  71       force_repaint_(false),
  72       current_shader_program_texture_target_(0),
  73       shader_program_(0),
  74       shader_texcoord_scale_location_(0),
  75       callback_factory_(this) {
  76 }
  77
  78 PepperVideoRenderer3D::~PepperVideoRenderer3D() {
  79   if (shader_program_)
  80     gles2_if_->DeleteProgram(graphics_.pp_resource(), shader_program_);
  81
  82   STLDeleteElements(&pending_packets_);
  83 }
  84
  85 bool PepperVideoRenderer3D::Initialize(pp::Instance* instance,
  86                                        const ClientContext& context,
  87                                        EventHandler* event_handler) {
  88   DCHECK(event_handler);
  89   DCHECK(!event_handler_);
  90
  91   event_handler_ = event_handler;
  92
  93   const int32_t context_attributes[] = {
  94       PP_GRAPHICS3DATTRIB_ALPHA_SIZE,     8,
  95       PP_GRAPHICS3DATTRIB_BLUE_SIZE,      8,
  96       PP_GRAPHICS3DATTRIB_GREEN_SIZE,     8,
  97       PP_GRAPHICS3DATTRIB_RED_SIZE,       8,
  98       PP_GRAPHICS3DATTRIB_DEPTH_SIZE,     0,
  99       PP_GRAPHICS3DATTRIB_STENCIL_SIZE,   0,
 100       PP_GRAPHICS3DATTRIB_SAMPLES,        0,
 101       PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
 102       PP_GRAPHICS3DATTRIB_WIDTH,          640,
 103       PP_GRAPHICS3DATTRIB_HEIGHT,         480,
 104       PP_GRAPHICS3DATTRIB_NONE,
 105   };
 106   graphics_ = pp::Graphics3D(instance, context_attributes);
 107
 108   if (graphics_.is_null()) {
 109     LOG(WARNING) << "Graphics3D interface is not available.";
 110     return false;
 111   }
 112   if (!instance->BindGraphics(graphics_)) {
 113     LOG(WARNING) << "Failed to bind Graphics3D.";
 114     return false;
 115   }
 116
 117   // Fetch the GLES2 interface to use to render frames.
 118   gles2_if_ = static_cast<const PPB_OpenGLES2*>(
 119       pp::Module::Get()->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
 120   CHECK(gles2_if_);
 121
 122   video_decoder_ = pp::VideoDecoder(instance);
 123   if (video_decoder_.is_null()) {
 124     LOG(WARNING) << "VideoDecoder interface is not available.";
 125     return false;
 126   }
 127
 128   PP_Resource graphics_3d = graphics_.pp_resource();
 129
 130   gles2_if_->ClearColor(graphics_3d, 1, 0, 0, 1);
 131   gles2_if_->Clear(graphics_3d, GL_COLOR_BUFFER_BIT);
 132
 133   // Assign vertex positions and texture coordinates to buffers for use in
 134   // shader program.
 135   static const float kVertices[] = {
 136       -1, -1, -1, 1, 1, -1, 1, 1,  // Position coordinates.
 137       0,  1,  0,  0, 1, 1,  1, 0,  // Texture coordinates.
 138   };
 139
 140   GLuint buffer;
 141   gles2_if_->GenBuffers(graphics_3d, 1, &buffer);
 142   gles2_if_->BindBuffer(graphics_3d, GL_ARRAY_BUFFER, buffer);
 143   gles2_if_->BufferData(graphics_3d, GL_ARRAY_BUFFER, sizeof(kVertices),
 144                         kVertices, GL_STATIC_DRAW);
 145
 146   CheckGLError();
 147
 148   return true;
 149 }
 150
 151 void PepperVideoRenderer3D::OnViewChanged(const pp::View& view) {
 152   pp::Size size = view.GetRect().size();
 153   float scale = view.GetDeviceScale();
 154   view_size_.set(ceilf(size.width() * scale), ceilf(size.height() * scale));
 155   graphics_.ResizeBuffers(view_size_.width(), view_size_.height());
 156
 157   force_repaint_ = true;
 158   PaintIfNeeded();
 159 }
 160
 161 void PepperVideoRenderer3D::EnableDebugDirtyRegion(bool enable) {
 162   debug_dirty_region_ = enable;
 163 }
 164
 165 void PepperVideoRenderer3D::OnSessionConfig(
 166     const protocol::SessionConfig& config) {
 167   PP_VideoProfile video_profile = PP_VIDEOPROFILE_VP8_ANY;
 168   switch (config.video_config().codec) {
 169     case protocol::ChannelConfig::CODEC_VP8:
 170       video_profile = PP_VIDEOPROFILE_VP8_ANY;
 171       break;
 172     case protocol::ChannelConfig::CODEC_VP9:
 173       video_profile = PP_VIDEOPROFILE_VP9_ANY;
 174       break;
 175     default:
 176       NOTREACHED();
 177   }
 178
 179   bool supports_video_decoder_1_1 =
 180       pp::Module::Get()->GetBrowserInterface(
 181           PPB_VIDEODECODER_INTERFACE_1_1) != NULL;
 182   int32_t result = video_decoder_.Initialize(
 183       graphics_, video_profile, PP_HARDWAREACCELERATION_WITHFALLBACK,
 184       supports_video_decoder_1_1 ? kMinimumPictureCount : 0,
 185       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnInitialized));
 186   CHECK_EQ(result, PP_OK_COMPLETIONPENDING)
 187       << "video_decoder_.Initialize() returned " << result;
 188 }
 189
 190 ChromotingStats* PepperVideoRenderer3D::GetStats() {
 191   return &stats_;
 192 }
 193
 194 protocol::VideoStub* PepperVideoRenderer3D::GetVideoStub() {
 195   return this;
 196 }
 197
 198 void PepperVideoRenderer3D::ProcessVideoPacket(scoped_ptr<VideoPacket> packet,
 199                                                const base::Closure& done) {
 200   base::ScopedClosureRunner done_runner(done);
 201
 202   stats_.RecordVideoPacketStats(*packet);
 203
 204   // Don't need to do anything if the packet is empty. Host sends empty video
 205   // packets when the screen is not changing.
 206   if (!packet->data().size())
 207     return;
 208
 209   bool resolution_changed = false;
 210
 211   if (packet->format().has_screen_width() &&
 212       packet->format().has_screen_height()) {
 213     webrtc::DesktopSize frame_size(packet->format().screen_width(),
 214                                    packet->format().screen_height());
 215     if (!frame_size_.equals(frame_size)) {
 216       frame_size_ = frame_size;
 217       resolution_changed = true;
 218     }
 219   }
 220
 221   if (packet->format().has_x_dpi() && packet->format().has_y_dpi()) {
 222     webrtc::DesktopVector frame_dpi(packet->format().x_dpi(),
 223                                     packet->format().y_dpi());
 224     if (!frame_dpi_.equals(frame_dpi)) {
 225       frame_dpi_ = frame_dpi;
 226       resolution_changed = true;
 227     }
 228   }
 229
 230   if (resolution_changed)
 231     event_handler_->OnVideoSize(frame_size_, frame_dpi_);
 232
 233   // Process the frame shape, if supplied.
 234   if (packet->has_use_desktop_shape()) {
 235     if (packet->use_desktop_shape()) {
 236       scoped_ptr<webrtc::DesktopRegion> shape(new webrtc::DesktopRegion);
 237       for (int i = 0; i < packet->desktop_shape_rects_size(); ++i) {
 238         Rect remoting_rect = packet->desktop_shape_rects(i);
 239         shape->AddRect(webrtc::DesktopRect::MakeXYWH(
 240             remoting_rect.x(), remoting_rect.y(), remoting_rect.width(),
 241             remoting_rect.height()));
 242       }
 243       if (!frame_shape_ || !frame_shape_->Equals(*shape)) {
 244         frame_shape_ = shape.Pass();
 245         event_handler_->OnVideoShape(frame_shape_.get());
 246       }
 247     } else if (frame_shape_) {
 248       frame_shape_ = nullptr;
 249       event_handler_->OnVideoShape(nullptr);
 250     }
 251   }
 252
 253   // Report the dirty region, for debugging, if requested.
 254   if (debug_dirty_region_) {
 255     webrtc::DesktopRegion dirty_region;
 256     for (int i = 0; i < packet->dirty_rects_size(); ++i) {
 257       Rect remoting_rect = packet->dirty_rects(i);
 258       dirty_region.AddRect(webrtc::DesktopRect::MakeXYWH(
 259           remoting_rect.x(), remoting_rect.y(),
 260           remoting_rect.width(), remoting_rect.height()));
 261     }
 262     event_handler_->OnVideoFrameDirtyRegion(dirty_region);
 263   }
 264
 265   pending_packets_.push_back(
 266       new PendingPacket(packet.Pass(), done_runner.Release()));
 267   DecodeNextPacket();
 268 }
 269
 270 void PepperVideoRenderer3D::OnInitialized(int32_t result) {
 271   // Assume that VP8 and VP9 codecs are always supported by the browser.
 272   CHECK_EQ(result, PP_OK) << "VideoDecoder::Initialize() failed: " << result;
 273   initialization_finished_ = true;
 274
 275   // Start decoding in case a frame was received during decoder initialization.
 276   DecodeNextPacket();
 277 }
 278
 279 void PepperVideoRenderer3D::DecodeNextPacket() {
 280   if (!initialization_finished_ || decode_pending_ || pending_packets_.empty())
 281     return;
 282
 283   ++latest_frame_id_;
 284   frame_decode_timestamps_.push_back(
 285       FrameDecodeTimestamp(latest_frame_id_, base::TimeTicks::Now()));
 286
 287   const VideoPacket* packet = pending_packets_.front()->packet();
 288
 289   int32_t result = video_decoder_.Decode(
 290       latest_frame_id_, packet->data().size(), packet->data().data(),
 291       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnDecodeDone));
 292   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 293   decode_pending_ = true;
 294 }
 295
 296 void PepperVideoRenderer3D::OnDecodeDone(int32_t result) {
 297   DCHECK(decode_pending_);
 298   decode_pending_ = false;
 299
 300   if (result != PP_OK) {
 301     LOG(ERROR) << "VideoDecoder::Decode() returned " << result;
 302     event_handler_->OnVideoDecodeError();
 303     return;
 304   }
 305
 306   delete pending_packets_.front();
 307   pending_packets_.pop_front();
 308
 309   DecodeNextPacket();
 310   GetNextPicture();
 311 }
 312
 313 void PepperVideoRenderer3D::GetNextPicture() {
 314   if (get_picture_pending_)
 315     return;
 316
 317   int32_t result =
 318       video_decoder_.GetPicture(callback_factory_.NewCallbackWithOutput(
 319           &PepperVideoRenderer3D::OnPictureReady));
 320   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 321   get_picture_pending_ = true;
 322 }
 323
 324 void PepperVideoRenderer3D::OnPictureReady(int32_t result,
 325                                            PP_VideoPicture picture) {
 326   DCHECK(get_picture_pending_);
 327   get_picture_pending_ = false;
 328
 329   if (result != PP_OK) {
 330     LOG(ERROR) << "VideoDecoder::GetPicture() returned " << result;
 331     event_handler_->OnVideoDecodeError();
 332     return;
 333   }
 334
 335   CHECK(!frame_decode_timestamps_.empty());
 336   const FrameDecodeTimestamp& frame_timer = frame_decode_timestamps_.front();
 337
 338   if (picture.decode_id != frame_timer.frame_id) {
 339     LOG(ERROR)
 340         << "Received a video packet that didn't contain a complete frame.";
 341     event_handler_->OnVideoDecodeError();
 342     return;
 343   }
 344
 345   base::TimeDelta decode_time =
 346       base::TimeTicks::Now() - frame_timer.decode_started_time;
 347   stats_.RecordDecodeTime(decode_time.InMilliseconds());
 348
 349   frame_decode_timestamps_.pop_front();
 350
 351   next_picture_.reset(new Picture(&video_decoder_, picture));
 352
 353   PaintIfNeeded();
 354   GetNextPicture();
 355 }
 356
 357 void PepperVideoRenderer3D::PaintIfNeeded() {
 358   bool need_repaint = next_picture_ || (force_repaint_ && current_picture_);
 359   if (paint_pending_ || !need_repaint)
 360     return;
 361
 362   if (next_picture_)
 363     current_picture_ = next_picture_.Pass();
 364
 365   force_repaint_ = false;
 366   latest_paint_started_time_ = base::TimeTicks::Now();
 367
 368   const PP_VideoPicture& picture = current_picture_->picture();
 369   PP_Resource graphics_3d = graphics_.pp_resource();
 370
 371   EnsureProgramForTexture(picture.texture_target);
 372
 373   gles2_if_->UseProgram(graphics_3d, shader_program_);
 374
 375   // Calculate v_scale passed to the vertex shader.
 376   double scale_x = picture.visible_rect.size.width;
 377   double scale_y = picture.visible_rect.size.height;
 378   if (picture.texture_target != GL_TEXTURE_RECTANGLE_ARB) {
 379     scale_x /= picture.texture_size.width;
 380     scale_y /= picture.texture_size.height;
 381   }
 382   gles2_if_->Uniform2f(graphics_3d, shader_texcoord_scale_location_,
 383                        scale_x, scale_y);
 384
 385   // Set viewport position & dimensions.
 386   gles2_if_->Viewport(graphics_3d, 0, 0, view_size_.width(),
 387                       view_size_.height());
 388
 389   // Select the texture unit GL_TEXTURE0.
 390   gles2_if_->ActiveTexture(graphics_3d, GL_TEXTURE0);
 391
 392   // Select the texture.
 393   gles2_if_->BindTexture(graphics_3d, picture.texture_target,
 394                          picture.texture_id);
 395
 396   // Select linear filter in case the texture needs to be scaled.
 397   gles2_if_->TexParameteri(graphics_3d, picture.texture_target,
 398                            GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 399
 400   // When view dimensions are a multiple of the frame size then use
 401   // nearest-neighbor scaling to achieve crisper image. Linear filter is used in
 402   // all other cases.
 403   GLint mag_filter = GL_LINEAR;
 404   if (view_size_.width() % picture.visible_rect.size.width == 0 &&
 405       view_size_.height() % picture.visible_rect.size.height == 0) {
 406     mag_filter = GL_NEAREST;
 407   }
 408   gles2_if_->TexParameteri(graphics_3d, picture.texture_target,
 409                            GL_TEXTURE_MAG_FILTER, mag_filter);
 410
 411   // Render texture by drawing a triangle strip with 4 vertices.
 412   gles2_if_->DrawArrays(graphics_3d, GL_TRIANGLE_STRIP, 0, 4);
 413
 414   CheckGLError();
 415
 416   // Request PPAPI display the queued texture.
 417   int32_t result = graphics_.SwapBuffers(
 418       callback_factory_.NewCallback(&PepperVideoRenderer3D::OnPaintDone));
 419   CHECK_EQ(result, PP_OK_COMPLETIONPENDING);
 420   paint_pending_ = true;
 421 }
 422
 423 void PepperVideoRenderer3D::OnPaintDone(int32_t result) {
 424   CHECK_EQ(result, PP_OK) << "Graphics3D::SwapBuffers() failed";
 425
 426   paint_pending_ = false;
 427   base::TimeDelta paint_time =
 428       base::TimeTicks::Now() - latest_paint_started_time_;
 429   stats_.RecordPaintTime(paint_time.InMilliseconds());
 430   PaintIfNeeded();
 431 }
 432
 433 void PepperVideoRenderer3D::EnsureProgramForTexture(uint32_t texture_target) {
 434   static const char kVertexShader[] =
 435       "varying vec2 v_texCoord;            \n"
 436       "attribute vec4 a_position;          \n"
 437       "attribute vec2 a_texCoord;          \n"
 438       "uniform vec2 v_scale;               \n"
 439       "void main()                         \n"
 440       "{                                   \n"
 441       "    v_texCoord = v_scale * a_texCoord; \n"
 442       "    gl_Position = a_position;       \n"
 443       "}";
 444
 445   static const char kFragmentShader2D[] =
 446       "precision mediump float;            \n"
 447       "varying vec2 v_texCoord;            \n"
 448       "uniform sampler2D s_texture;        \n"
 449       "void main()                         \n"
 450       "{"
 451       "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
 452       "}";
 453
 454   static const char kFragmentShaderRectangle[] =
 455       "#extension GL_ARB_texture_rectangle : require\n"
 456       "precision mediump float;            \n"
 457       "varying vec2 v_texCoord;            \n"
 458       "uniform sampler2DRect s_texture;    \n"
 459       "void main()                         \n"
 460       "{"
 461       "    gl_FragColor = texture2DRect(s_texture, v_texCoord).rgba; \n"
 462       "}";
 463
 464   static const char kFragmentShaderExternal[] =
 465       "#extension GL_OES_EGL_image_external : require\n"
 466       "precision mediump float;            \n"
 467       "varying vec2 v_texCoord;            \n"
 468       "uniform samplerExternalOES s_texture; \n"
 469       "void main()                         \n"
 470       "{"
 471       "    gl_FragColor = texture2D(s_texture, v_texCoord); \n"
 472       "}";
 473
 474   // Initialize shader program only if texture type has changed.
 475   if (current_shader_program_texture_target_ != texture_target) {
 476     current_shader_program_texture_target_ = texture_target;
 477
 478     if (texture_target == GL_TEXTURE_2D) {
 479       CreateProgram(kVertexShader, kFragmentShader2D);
 480     } else if (texture_target == GL_TEXTURE_RECTANGLE_ARB) {
 481       CreateProgram(kVertexShader, kFragmentShaderRectangle);
 482     } else if (texture_target == GL_TEXTURE_EXTERNAL_OES) {
 483       CreateProgram(kVertexShader, kFragmentShaderExternal);
 484     } else {
 485       LOG(FATAL) << "Unknown texture target: " << texture_target;
 486     }
 487   }
 488 }
 489
 490 void PepperVideoRenderer3D::CreateProgram(const char* vertex_shader,
 491                                           const char* fragment_shader) {
 492   PP_Resource graphics_3d = graphics_.pp_resource();
 493   if (shader_program_)
 494     gles2_if_->DeleteProgram(graphics_3d, shader_program_);
 495
 496   // Create shader program.
 497   shader_program_ = gles2_if_->CreateProgram(graphics_3d);
 498   CreateShaderProgram(GL_VERTEX_SHADER, vertex_shader);
 499   CreateShaderProgram(GL_FRAGMENT_SHADER, fragment_shader);
 500   gles2_if_->LinkProgram(graphics_3d, shader_program_);
 501   gles2_if_->UseProgram(graphics_3d, shader_program_);
 502   gles2_if_->Uniform1i(
 503       graphics_3d,
 504       gles2_if_->GetUniformLocation(graphics_3d, shader_program_, "s_texture"),
 505       0);
 506   CheckGLError();
 507
 508   shader_texcoord_scale_location_ = gles2_if_->GetUniformLocation(
 509       graphics_3d, shader_program_, "v_scale");
 510
 511   GLint pos_location = gles2_if_->GetAttribLocation(
 512       graphics_3d, shader_program_, "a_position");
 513   GLint tc_location = gles2_if_->GetAttribLocation(
 514       graphics_3d, shader_program_, "a_texCoord");
 515   CheckGLError();
 516
 517   // Construct the vertex array for DrawArrays(), using the buffer created in
 518   // Initialize().
 519   gles2_if_->EnableVertexAttribArray(graphics_3d, pos_location);
 520   gles2_if_->VertexAttribPointer(graphics_3d, pos_location, 2, GL_FLOAT,
 521                                  GL_FALSE, 0, 0);
 522   gles2_if_->EnableVertexAttribArray(graphics_3d, tc_location);
 523   gles2_if_->VertexAttribPointer(
 524       graphics_3d, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 525       static_cast<float*>(0) + 8);  // Skip position coordinates.
 526
 527   gles2_if_->UseProgram(graphics_3d, 0);
 528
 529   CheckGLError();
 530 }
 531
 532 void PepperVideoRenderer3D::CreateShaderProgram(int type, const char* source) {
 533   int size = strlen(source);
 534   GLuint shader = gles2_if_->CreateShader(graphics_.pp_resource(), type);
 535   gles2_if_->ShaderSource(graphics_.pp_resource(), shader, 1, &source, &size);
 536   gles2_if_->CompileShader(graphics_.pp_resource(), shader);
 537   gles2_if_->AttachShader(graphics_.pp_resource(), shader_program_, shader);
 538   gles2_if_->DeleteShader(graphics_.pp_resource(), shader);
 539 }
 540
 541 void PepperVideoRenderer3D::CheckGLError() {
 542   GLenum error = gles2_if_->GetError(graphics_.pp_resource());
 543   CHECK_EQ(error, static_cast<GLenum>(GL_NO_ERROR)) << "GL error: " << error;
 544 }
 545
 546 }  // namespace remoting