ppapi/examples/media_stream_video/media_stream_video.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <GLES2/gl2.h>
   6 #include <GLES2/gl2ext.h>
   7 #include <string.h>
   8
   9 #include <vector>
  10
  11 #include "ppapi/c/pp_errors.h"
  12 #include "ppapi/c/ppb_opengles2.h"
  13 #include "ppapi/cpp/completion_callback.h"
  14 #include "ppapi/cpp/graphics_3d.h"
  15 #include "ppapi/cpp/graphics_3d_client.h"
  16 #include "ppapi/cpp/instance.h"
  17 #include "ppapi/cpp/media_stream_video_track.h"
  18 #include "ppapi/cpp/module.h"
  19 #include "ppapi/cpp/rect.h"
  20 #include "ppapi/cpp/var.h"
  21 #include "ppapi/cpp/var_dictionary.h"
  22 #include "ppapi/cpp/video_frame.h"
  23 #include "ppapi/lib/gl/gles2/gl2ext_ppapi.h"
  24 #include "ppapi/utility/completion_callback_factory.h"
  25
  26 // When compiling natively on Windows, PostMessage can be #define-d to
  27 // something else.
  28 #ifdef PostMessage
  29 #undef PostMessage
  30 #endif
  31
  32 // Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
  33 // function to preserve line number information in the failure message.
  34 #define AssertNoGLError() \
  35   PP_DCHECK(!glGetError());
  36
  37 namespace {
  38
  39 // This object is the global object representing this plugin library as long
  40 // as it is loaded.
  41 class MediaStreamVideoModule : public pp::Module {
  42  public:
  43   MediaStreamVideoModule() : pp::Module() {}
  44   virtual ~MediaStreamVideoModule() {}
  45
  46   virtual pp::Instance* CreateInstance(PP_Instance instance);
  47 };
  48
  49 class MediaStreamVideoDemoInstance : public pp::Instance,
  50                         public pp::Graphics3DClient {
  51  public:
  52   MediaStreamVideoDemoInstance(PP_Instance instance, pp::Module* module);
  53   virtual ~MediaStreamVideoDemoInstance();
  54
  55   // pp::Instance implementation (see PPP_Instance).
  56   virtual void DidChangeView(const pp::Rect& position,
  57                              const pp::Rect& clip_ignored);
  58   virtual void HandleMessage(const pp::Var& message_data);
  59
  60   // pp::Graphics3DClient implementation.
  61   virtual void Graphics3DContextLost() {
  62     InitGL();
  63     CreateTextures();
  64     Render();
  65   }
  66
  67  private:
  68   void DrawYUV();
  69   void DrawRGB();
  70   void Render();
  71
  72   // GL-related functions.
  73   void InitGL();
  74   GLuint CreateTexture(int32_t width, int32_t height, int unit, bool rgba);
  75   void CreateGLObjects();
  76   void CreateShader(GLuint program, GLenum type, const char* source);
  77   void PaintFinished(int32_t result);
  78   void CreateTextures();
  79   void ConfigureTrack();
  80
  81
  82   // MediaStreamVideoTrack callbacks.
  83   void OnConfigure(int32_t result);
  84   void OnGetFrame(int32_t result, pp::VideoFrame frame);
  85
  86   pp::Size position_size_;
  87   bool is_painting_;
  88   bool needs_paint_;
  89   bool is_bgra_;
  90   GLuint program_yuv_;
  91   GLuint program_rgb_;
  92   GLuint buffer_;
  93   GLuint texture_y_;
  94   GLuint texture_u_;
  95   GLuint texture_v_;
  96   GLuint texture_rgb_;
  97   pp::MediaStreamVideoTrack video_track_;
  98   pp::CompletionCallbackFactory<MediaStreamVideoDemoInstance> callback_factory_;
  99   std::vector<int32_t> attrib_list_;
 100
 101   // MediaStreamVideoTrack attributes:
 102   bool need_config_;
 103   PP_VideoFrame_Format attrib_format_;
 104   int32_t attrib_width_;
 105   int32_t attrib_height_;
 106
 107   // Owned data.
 108   pp::Graphics3D* context_;
 109
 110   pp::Size frame_size_;
 111 };
 112
 113 MediaStreamVideoDemoInstance::MediaStreamVideoDemoInstance(
 114     PP_Instance instance, pp::Module* module)
 115     : pp::Instance(instance),
 116       pp::Graphics3DClient(this),
 117       is_painting_(false),
 118       needs_paint_(false),
 119       is_bgra_(false),
 120       texture_y_(0),
 121       texture_u_(0),
 122       texture_v_(0),
 123       texture_rgb_(0),
 124       callback_factory_(this),
 125       need_config_(false),
 126       attrib_format_(PP_VIDEOFRAME_FORMAT_I420),
 127       attrib_width_(0),
 128       attrib_height_(0),
 129       context_(NULL) {
 130   if (!glInitializePPAPI(pp::Module::Get()->get_browser_interface())) {
 131     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Unable to initialize GL PPAPI!"));
 132     assert(false);
 133   }
 134 }
 135
 136 MediaStreamVideoDemoInstance::~MediaStreamVideoDemoInstance() {
 137   delete context_;
 138 }
 139
 140 void MediaStreamVideoDemoInstance::DidChangeView(
 141     const pp::Rect& position, const pp::Rect& clip_ignored) {
 142   if (position.width() == 0 || position.height() == 0)
 143     return;
 144   if (position.size() == position_size_)
 145     return;
 146
 147   position_size_ = position.size();
 148
 149   // Initialize graphics.
 150   InitGL();
 151   Render();
 152 }
 153
 154 void MediaStreamVideoDemoInstance::HandleMessage(const pp::Var& var_message) {
 155   if (!var_message.is_dictionary()) {
 156     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
 157     return;
 158   }
 159
 160   pp::VarDictionary var_dictionary_message(var_message);
 161   std::string command = var_dictionary_message.Get("command").AsString();
 162
 163   if (command == "init") {
 164     pp::Var var_track = var_dictionary_message.Get("track");
 165     if (!var_track.is_resource())
 166       return;
 167     pp::Resource resource_track = var_track.AsResource();
 168     video_track_ = pp::MediaStreamVideoTrack(resource_track);
 169     ConfigureTrack();
 170   } else if (command == "format") {
 171     std::string str_format = var_dictionary_message.Get("format").AsString();
 172     if (str_format == "YV12") {
 173       attrib_format_ = PP_VIDEOFRAME_FORMAT_YV12;
 174     } else if (str_format == "I420") {
 175       attrib_format_ = PP_VIDEOFRAME_FORMAT_I420;
 176     } else if (str_format == "BGRA") {
 177       attrib_format_ = PP_VIDEOFRAME_FORMAT_BGRA;
 178     } else {
 179       attrib_format_ = PP_VIDEOFRAME_FORMAT_UNKNOWN;
 180     }
 181     need_config_ = true;
 182   } else if (command == "size") {
 183     attrib_width_ = var_dictionary_message.Get("width").AsInt();
 184     attrib_height_ = var_dictionary_message.Get("height").AsInt();
 185     need_config_ = true;
 186   } else {
 187     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
 188   }
 189 }
 190
 191 void MediaStreamVideoDemoInstance::InitGL() {
 192   PP_DCHECK(position_size_.width() && position_size_.height());
 193   is_painting_ = false;
 194
 195   delete context_;
 196   int32_t attributes[] = {
 197     PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
 198     PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
 199     PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
 200     PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
 201     PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
 202     PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
 203     PP_GRAPHICS3DATTRIB_SAMPLES, 0,
 204     PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
 205     PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
 206     PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
 207     PP_GRAPHICS3DATTRIB_NONE,
 208   };
 209   context_ = new pp::Graphics3D(this, attributes);
 210   PP_DCHECK(!context_->is_null());
 211
 212   glSetCurrentContextPPAPI(context_->pp_resource());
 213
 214   // Set viewport window size and clear color bit.
 215   glClearColor(1, 0, 0, 1);
 216   glClear(GL_COLOR_BUFFER_BIT);
 217   glViewport(0, 0, position_size_.width(), position_size_.height());
 218
 219   BindGraphics(*context_);
 220   AssertNoGLError();
 221
 222   CreateGLObjects();
 223 }
 224
 225 void MediaStreamVideoDemoInstance::DrawYUV() {
 226   static const float kColorMatrix[9] = {
 227     1.1643828125f, 1.1643828125f, 1.1643828125f,
 228     0.0f, -0.39176171875f, 2.017234375f,
 229     1.59602734375f, -0.81296875f, 0.0f
 230   };
 231
 232   glUseProgram(program_yuv_);
 233   glUniform1i(glGetUniformLocation(program_yuv_, "y_texture"), 0);
 234   glUniform1i(glGetUniformLocation(program_yuv_, "u_texture"), 1);
 235   glUniform1i(glGetUniformLocation(program_yuv_, "v_texture"), 2);
 236   glUniformMatrix3fv(glGetUniformLocation(program_yuv_, "color_matrix"),
 237       1, GL_FALSE, kColorMatrix);
 238   AssertNoGLError();
 239
 240   GLint pos_location = glGetAttribLocation(program_yuv_, "a_position");
 241   GLint tc_location = glGetAttribLocation(program_yuv_, "a_texCoord");
 242   AssertNoGLError();
 243   glEnableVertexAttribArray(pos_location);
 244   glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
 245   glEnableVertexAttribArray(tc_location);
 246   glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 247       static_cast<float*>(0) + 16);  // Skip position coordinates.
 248   AssertNoGLError();
 249
 250   glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
 251   AssertNoGLError();
 252 }
 253
 254 void MediaStreamVideoDemoInstance::DrawRGB() {
 255   glUseProgram(program_rgb_);
 256   glUniform1i(glGetUniformLocation(program_rgb_, "rgb_texture"), 3);
 257   AssertNoGLError();
 258
 259   GLint pos_location = glGetAttribLocation(program_rgb_, "a_position");
 260   GLint tc_location = glGetAttribLocation(program_rgb_, "a_texCoord");
 261   AssertNoGLError();
 262   glEnableVertexAttribArray(pos_location);
 263   glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, 0, 0);
 264   glEnableVertexAttribArray(tc_location);
 265   glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 266       static_cast<float*>(0) + 16);  // Skip position coordinates.
 267   AssertNoGLError();
 268
 269   glDrawArrays(GL_TRIANGLE_STRIP, 4, 4);
 270 }
 271
 272 void MediaStreamVideoDemoInstance::Render() {
 273   PP_DCHECK(!is_painting_);
 274   is_painting_ = true;
 275   needs_paint_ = false;
 276
 277   if (texture_y_) {
 278     DrawRGB();
 279     DrawYUV();
 280   } else {
 281     glClear(GL_COLOR_BUFFER_BIT);
 282   }
 283   pp::CompletionCallback cb = callback_factory_.NewCallback(
 284       &MediaStreamVideoDemoInstance::PaintFinished);
 285   context_->SwapBuffers(cb);
 286 }
 287
 288 void MediaStreamVideoDemoInstance::PaintFinished(int32_t result) {
 289   is_painting_ = false;
 290   if (needs_paint_)
 291     Render();
 292 }
 293
 294 GLuint MediaStreamVideoDemoInstance::CreateTexture(
 295     int32_t width, int32_t height, int unit, bool rgba) {
 296   GLuint texture_id;
 297   glGenTextures(1, &texture_id);
 298   AssertNoGLError();
 299
 300   // Assign parameters.
 301   glActiveTexture(GL_TEXTURE0 + unit);
 302   glBindTexture(GL_TEXTURE_2D, texture_id);
 303   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
 304   glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 305   glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 306   glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 307   // Allocate texture.
 308   glTexImage2D(GL_TEXTURE_2D, 0,
 309                rgba ? GL_BGRA_EXT : GL_LUMINANCE,
 310                width, height, 0,
 311                rgba ? GL_BGRA_EXT : GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
 312   AssertNoGLError();
 313   return texture_id;
 314 }
 315
 316 void MediaStreamVideoDemoInstance::CreateGLObjects() {
 317   // Code and constants for shader.
 318   static const char kVertexShader[] =
 319       "varying vec2 v_texCoord;            \n"
 320       "attribute vec4 a_position;          \n"
 321       "attribute vec2 a_texCoord;          \n"
 322       "void main()                         \n"
 323       "{                                   \n"
 324       "    v_texCoord = a_texCoord;        \n"
 325       "    gl_Position = a_position;       \n"
 326       "}";
 327
 328   static const char kFragmentShaderYUV[] =
 329       "precision mediump float;                                   \n"
 330       "varying vec2 v_texCoord;                                   \n"
 331       "uniform sampler2D y_texture;                               \n"
 332       "uniform sampler2D u_texture;                               \n"
 333       "uniform sampler2D v_texture;                               \n"
 334       "uniform mat3 color_matrix;                                 \n"
 335       "void main()                                                \n"
 336       "{                                                          \n"
 337       "  vec3 yuv;                                                \n"
 338       "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
 339       "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
 340       "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
 341       "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
 342       "  gl_FragColor = vec4(rgb, 1.0);                           \n"
 343       "}";
 344
 345   static const char kFragmentShaderRGB[] =
 346       "precision mediump float;                                   \n"
 347       "varying vec2 v_texCoord;                                   \n"
 348       "uniform sampler2D rgb_texture;                             \n"
 349       "void main()                                                \n"
 350       "{                                                          \n"
 351       "  gl_FragColor = texture2D(rgb_texture, v_texCoord);       \n"
 352       "}";
 353
 354   // Create shader programs.
 355   program_yuv_ = glCreateProgram();
 356   CreateShader(program_yuv_, GL_VERTEX_SHADER, kVertexShader);
 357   CreateShader(program_yuv_, GL_FRAGMENT_SHADER, kFragmentShaderYUV);
 358   glLinkProgram(program_yuv_);
 359   AssertNoGLError();
 360
 361   program_rgb_ = glCreateProgram();
 362   CreateShader(program_rgb_, GL_VERTEX_SHADER, kVertexShader);
 363   CreateShader(program_rgb_, GL_FRAGMENT_SHADER, kFragmentShaderRGB);
 364   glLinkProgram(program_rgb_);
 365   AssertNoGLError();
 366
 367   // Assign vertex positions and texture coordinates to buffers for use in
 368   // shader program.
 369   static const float kVertices[] = {
 370     -1, 1, -1, -1, 0, 1, 0, -1,  // Position coordinates.
 371     0, 1, 0, -1, 1, 1, 1, -1,  // Position coordinates.
 372     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
 373     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
 374   };
 375
 376   glGenBuffers(1, &buffer_);
 377   glBindBuffer(GL_ARRAY_BUFFER, buffer_);
 378   glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW);
 379   AssertNoGLError();
 380 }
 381
 382 void MediaStreamVideoDemoInstance::CreateShader(
 383     GLuint program, GLenum type, const char* source) {
 384   GLuint shader = glCreateShader(type);
 385   GLint length = static_cast<GLint>(strlen(source) + 1);
 386   glShaderSource(shader, 1, &source, &length);
 387   glCompileShader(shader);
 388   glAttachShader(program, shader);
 389   glDeleteShader(shader);
 390 }
 391
 392 void MediaStreamVideoDemoInstance::CreateTextures() {
 393   int32_t width = frame_size_.width();
 394   int32_t height = frame_size_.height();
 395   if (width == 0 || height == 0)
 396     return;
 397   if (texture_y_)
 398     glDeleteTextures(1, &texture_y_);
 399   if (texture_u_)
 400     glDeleteTextures(1, &texture_u_);
 401   if (texture_v_)
 402     glDeleteTextures(1, &texture_v_);
 403   if (texture_rgb_)
 404     glDeleteTextures(1, &texture_rgb_);
 405   texture_y_ = CreateTexture(width, height, 0, false);
 406
 407   texture_u_ = CreateTexture(width / 2, height / 2, 1, false);
 408   texture_v_ = CreateTexture(width / 2, height / 2, 2, false);
 409   texture_rgb_ = CreateTexture(width, height, 3, true);
 410 }
 411
 412 void MediaStreamVideoDemoInstance::ConfigureTrack() {
 413   const int32_t attrib_list[] = {
 414       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT, attrib_format_,
 415       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH, attrib_width_,
 416       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT, attrib_height_,
 417       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE
 418     };
 419   video_track_.Configure(attrib_list, callback_factory_.NewCallback(
 420         &MediaStreamVideoDemoInstance::OnConfigure));
 421 }
 422
 423 void MediaStreamVideoDemoInstance::OnConfigure(int32_t result) {
 424   video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 425       &MediaStreamVideoDemoInstance::OnGetFrame));
 426 }
 427
 428 void MediaStreamVideoDemoInstance::OnGetFrame(
 429     int32_t result, pp::VideoFrame frame) {
 430   if (result != PP_OK)
 431     return;
 432   const char* data = static_cast<const char*>(frame.GetDataBuffer());
 433   pp::Size size;
 434   frame.GetSize(&size);
 435
 436   if (size != frame_size_) {
 437     frame_size_ = size;
 438     CreateTextures();
 439   }
 440
 441   is_bgra_ = (frame.GetFormat() == PP_VIDEOFRAME_FORMAT_BGRA);
 442
 443   int32_t width = frame_size_.width();
 444   int32_t height = frame_size_.height();
 445   if (!is_bgra_) {
 446     glActiveTexture(GL_TEXTURE0);
 447     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
 448                     GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 449
 450     data += width * height;
 451     width /= 2;
 452     height /= 2;
 453
 454     glActiveTexture(GL_TEXTURE1);
 455     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
 456                     GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 457
 458     data += width * height;
 459     glActiveTexture(GL_TEXTURE2);
 460     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
 461                     GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 462   } else {
 463     glActiveTexture(GL_TEXTURE3);
 464     glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height,
 465                     GL_BGRA_EXT, GL_UNSIGNED_BYTE, data);
 466   }
 467
 468   if (is_painting_)
 469     needs_paint_ = true;
 470   else
 471     Render();
 472
 473   video_track_.RecycleFrame(frame);
 474   if (need_config_) {
 475     ConfigureTrack();
 476     need_config_ = false;
 477   } else {
 478     video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 479         &MediaStreamVideoDemoInstance::OnGetFrame));
 480   }
 481 }
 482
 483 pp::Instance* MediaStreamVideoModule::CreateInstance(PP_Instance instance) {
 484   return new MediaStreamVideoDemoInstance(instance, this);
 485 }
 486
 487 }  // anonymous namespace
 488
 489 namespace pp {
 490 // Factory function for your specialization of the Module object.
 491 Module* CreateModule() {
 492   return new MediaStreamVideoModule();
 493 }
 494 }  // namespace pp