ppapi/examples/media_stream_video/media_stream_video.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <vector>
   6
   7 #include "ppapi/c/pp_errors.h"
   8 #include "ppapi/c/ppb_opengles2.h"
   9 #include "ppapi/cpp/completion_callback.h"
  10 #include "ppapi/cpp/graphics_3d.h"
  11 #include "ppapi/cpp/graphics_3d_client.h"
  12 #include "ppapi/cpp/instance.h"
  13 #include "ppapi/cpp/media_stream_video_track.h"
  14 #include "ppapi/cpp/module.h"
  15 #include "ppapi/cpp/rect.h"
  16 #include "ppapi/cpp/var.h"
  17 #include "ppapi/cpp/video_frame.h"
  18 #include "ppapi/lib/gl/include/GLES2/gl2.h"
  19 #include "ppapi/lib/gl/include/GLES2/gl2ext.h"
  20 #include "ppapi/utility/completion_callback_factory.h"
  21
  22 // When compiling natively on Windows, PostMessage can be #define-d to
  23 // something else.
  24 #ifdef PostMessage
  25 #undef PostMessage
  26 #endif
  27
  28 // Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
  29 // function to preserve line number information in the failure message.
  30 #define AssertNoGLError() \
  31   PP_DCHECK(!gles2_if_->GetError(context_->pp_resource()));
  32
  33 namespace {
  34
  35 // This object is the global object representing this plugin library as long
  36 // as it is loaded.
  37 class MediaStreamVideoModule : public pp::Module {
  38  public:
  39   MediaStreamVideoModule() : pp::Module() {}
  40   virtual ~MediaStreamVideoModule() {}
  41
  42   virtual pp::Instance* CreateInstance(PP_Instance instance);
  43 };
  44
  45 class MediaStreamVideoDemoInstance : public pp::Instance,
  46                         public pp::Graphics3DClient {
  47  public:
  48   MediaStreamVideoDemoInstance(PP_Instance instance, pp::Module* module);
  49   virtual ~MediaStreamVideoDemoInstance();
  50
  51   // pp::Instance implementation (see PPP_Instance).
  52   virtual void DidChangeView(const pp::Rect& position,
  53                              const pp::Rect& clip_ignored);
  54   virtual void HandleMessage(const pp::Var& message_data);
  55
  56   // pp::Graphics3DClient implementation.
  57   virtual void Graphics3DContextLost() {
  58     InitGL();
  59     CreateTextures();
  60     Render();
  61   }
  62
  63  private:
  64   void DrawYUV();
  65   void DrawRGB();
  66   void Render();
  67
  68   // GL-related functions.
  69   void InitGL();
  70   GLuint CreateTexture(int32_t width, int32_t height, int unit, bool rgba);
  71   void CreateGLObjects();
  72   void CreateShader(GLuint program, GLenum type, const char* source, int size);
  73   void PaintFinished(int32_t result);
  74   void CreateTextures();
  75   void ConfigureTrack();
  76
  77
  78   // MediaStreamVideoTrack callbacks.
  79   void OnConfigure(int32_t result);
  80   void OnGetFrame(int32_t result, pp::VideoFrame frame);
  81
  82   pp::Size position_size_;
  83   bool is_painting_;
  84   bool needs_paint_;
  85   bool is_bgra_;
  86   GLuint program_yuv_;
  87   GLuint program_rgb_;
  88   GLuint buffer_;
  89   GLuint texture_y_;
  90   GLuint texture_u_;
  91   GLuint texture_v_;
  92   GLuint texture_rgb_;
  93   pp::MediaStreamVideoTrack video_track_;
  94   pp::CompletionCallbackFactory<MediaStreamVideoDemoInstance> callback_factory_;
  95   std::vector<int32_t> attrib_list_;
  96
  97   // MediaStreamVideoTrack attributes:
  98   bool need_config_;
  99   PP_VideoFrame_Format attrib_format_;
 100   int32_t attrib_width_;
 101   int32_t attrib_height_;
 102
 103   // Unowned pointers.
 104   const struct PPB_OpenGLES2* gles2_if_;
 105
 106   // Owned data.
 107   pp::Graphics3D* context_;
 108
 109   pp::Size frame_size_;
 110 };
 111
 112 MediaStreamVideoDemoInstance::MediaStreamVideoDemoInstance(
 113     PP_Instance instance, pp::Module* module)
 114     : pp::Instance(instance),
 115       pp::Graphics3DClient(this),
 116       is_painting_(false),
 117       needs_paint_(false),
 118       is_bgra_(false),
 119       texture_y_(0),
 120       texture_u_(0),
 121       texture_v_(0),
 122       texture_rgb_(0),
 123       callback_factory_(this),
 124       need_config_(false),
 125       attrib_format_(PP_VIDEOFRAME_FORMAT_I420),
 126       attrib_width_(0),
 127       attrib_height_(0),
 128       context_(NULL) {
 129   gles2_if_ = static_cast<const struct PPB_OpenGLES2*>(
 130       module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
 131   PP_DCHECK(gles2_if_);
 132 }
 133
 134 MediaStreamVideoDemoInstance::~MediaStreamVideoDemoInstance() {
 135   delete context_;
 136 }
 137
 138 void MediaStreamVideoDemoInstance::DidChangeView(
 139     const pp::Rect& position, const pp::Rect& clip_ignored) {
 140   if (position.width() == 0 || position.height() == 0)
 141     return;
 142   if (position.size() == position_size_)
 143     return;
 144
 145   position_size_ = position.size();
 146
 147   // Initialize graphics.
 148   InitGL();
 149   Render();
 150 }
 151
 152 void MediaStreamVideoDemoInstance::HandleMessage(const pp::Var& var_message) {
 153   if (!var_message.is_dictionary())
 154     return;
 155
 156   pp::VarDictionary var_dictionary_message(var_message);
 157   std::string command = var_dictionary_message.Get("command").AsString();
 158
 159   if (command == "init") {
 160     pp::Var var_track = var_dictionary_message.Get("track");
 161     if (!var_track.is_resource())
 162       return;
 163     pp::Resource resource_track = var_track.AsResource();
 164     video_track_ = pp::MediaStreamVideoTrack(resource_track);
 165     ConfigureTrack();
 166   } else if (command == "format") {
 167     std::string str_format =  var_dictionary_message.Get("format").AsString();
 168     if (str_format == "YV12") {
 169       attrib_format_ = PP_VIDEOFRAME_FORMAT_YV12;
 170     } else if (str_format == "I420") {
 171       attrib_format_ = PP_VIDEOFRAME_FORMAT_I420;
 172     } else if (str_format == "BGRA") {
 173       attrib_format_ = PP_VIDEOFRAME_FORMAT_BGRA;
 174     } else {
 175       attrib_format_ = PP_VIDEOFRAME_FORMAT_UNKNOWN;
 176     }
 177     need_config_ = true;
 178   } else if (command == "size") {
 179     attrib_width_ = var_dictionary_message.Get("width").AsInt();
 180     attrib_height_ = var_dictionary_message.Get("height").AsInt();
 181     need_config_ = true;
 182   }
 183 }
 184
 185 void MediaStreamVideoDemoInstance::InitGL() {
 186   PP_DCHECK(position_size_.width() && position_size_.height());
 187   is_painting_ = false;
 188
 189   delete context_;
 190   int32_t attributes[] = {
 191     PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
 192     PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
 193     PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
 194     PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
 195     PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
 196     PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
 197     PP_GRAPHICS3DATTRIB_SAMPLES, 0,
 198     PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
 199     PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
 200     PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
 201     PP_GRAPHICS3DATTRIB_NONE,
 202   };
 203   context_ = new pp::Graphics3D(this, attributes);
 204   PP_DCHECK(!context_->is_null());
 205
 206   // Set viewport window size and clear color bit.
 207   gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
 208   gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
 209   gles2_if_->Viewport(context_->pp_resource(), 0, 0,
 210                       position_size_.width(), position_size_.height());
 211
 212   BindGraphics(*context_);
 213   AssertNoGLError();
 214
 215   CreateGLObjects();
 216 }
 217
 218 void MediaStreamVideoDemoInstance::DrawYUV() {
 219   PP_Resource context = context_->pp_resource();
 220   static const float kColorMatrix[9] = {
 221     1.1643828125f, 1.1643828125f, 1.1643828125f,
 222     0.0f, -0.39176171875f, 2.017234375f,
 223     1.59602734375f, -0.81296875f, 0.0f
 224   };
 225
 226   gles2_if_->UseProgram(context, program_yuv_);
 227   gles2_if_->Uniform1i(context, gles2_if_->GetUniformLocation(
 228         context, program_yuv_, "y_texture"), 0);
 229   gles2_if_->Uniform1i(context, gles2_if_->GetUniformLocation(
 230         context, program_yuv_, "u_texture"), 1);
 231   gles2_if_->Uniform1i(context, gles2_if_->GetUniformLocation(
 232         context, program_yuv_, "v_texture"), 2);
 233   gles2_if_->UniformMatrix3fv(
 234       context,
 235       gles2_if_->GetUniformLocation(context, program_yuv_, "color_matrix"),
 236       1, GL_FALSE, kColorMatrix);
 237   AssertNoGLError();
 238
 239   GLint pos_location = gles2_if_->GetAttribLocation(
 240       context, program_yuv_, "a_position");
 241   GLint tc_location = gles2_if_->GetAttribLocation(
 242       context, program_yuv_, "a_texCoord");
 243   AssertNoGLError();
 244   gles2_if_->EnableVertexAttribArray(context, pos_location);
 245   gles2_if_->VertexAttribPointer(context, pos_location, 2,
 246                                  GL_FLOAT, GL_FALSE, 0, 0);
 247   gles2_if_->EnableVertexAttribArray(context, tc_location);
 248   gles2_if_->VertexAttribPointer(
 249       context, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 250       static_cast<float*>(0) + 16);  // Skip position coordinates.
 251   AssertNoGLError();
 252
 253   gles2_if_->DrawArrays(context, GL_TRIANGLE_STRIP, 0, 4);
 254   AssertNoGLError();
 255 }
 256
 257 void MediaStreamVideoDemoInstance::DrawRGB() {
 258   PP_Resource context = context_->pp_resource();
 259   gles2_if_->UseProgram(context, program_rgb_);
 260   gles2_if_->Uniform1i(context,
 261       gles2_if_->GetUniformLocation(context, program_rgb_, "rgb_texture"), 3);
 262   AssertNoGLError();
 263
 264   GLint pos_location = gles2_if_->GetAttribLocation(
 265       context, program_rgb_, "a_position");
 266   GLint tc_location = gles2_if_->GetAttribLocation(
 267       context, program_rgb_, "a_texCoord");
 268   AssertNoGLError();
 269   gles2_if_->EnableVertexAttribArray(context, pos_location);
 270   gles2_if_->VertexAttribPointer(context, pos_location, 2,
 271                                  GL_FLOAT, GL_FALSE, 0, 0);
 272   gles2_if_->EnableVertexAttribArray(context, tc_location);
 273   gles2_if_->VertexAttribPointer(
 274       context, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 275       static_cast<float*>(0) + 16);  // Skip position coordinates.
 276   AssertNoGLError();
 277
 278   gles2_if_->DrawArrays(context, GL_TRIANGLE_STRIP, 4, 4);
 279 }
 280
 281 void MediaStreamVideoDemoInstance::Render() {
 282   PP_DCHECK(!is_painting_);
 283   is_painting_ = true;
 284   needs_paint_ = false;
 285
 286   if (texture_y_) {
 287     DrawRGB();
 288     DrawYUV();
 289   } else {
 290     gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
 291   }
 292   pp::CompletionCallback cb = callback_factory_.NewCallback(
 293       &MediaStreamVideoDemoInstance::PaintFinished);
 294   context_->SwapBuffers(cb);
 295 }
 296
 297 void MediaStreamVideoDemoInstance::PaintFinished(int32_t result) {
 298   is_painting_ = false;
 299   if (needs_paint_)
 300     Render();
 301 }
 302
 303 GLuint MediaStreamVideoDemoInstance::CreateTexture(
 304     int32_t width, int32_t height, int unit, bool rgba) {
 305   GLuint texture_id;
 306   gles2_if_->GenTextures(context_->pp_resource(), 1, &texture_id);
 307   AssertNoGLError();
 308
 309   // Assign parameters.
 310   gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0 + unit);
 311   gles2_if_->BindTexture(context_->pp_resource(), GL_TEXTURE_2D, texture_id);
 312   gles2_if_->TexParameteri(
 313       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
 314       GL_NEAREST);
 315   gles2_if_->TexParameteri(
 316       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
 317       GL_NEAREST);
 318   gles2_if_->TexParameterf(
 319       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
 320       GL_CLAMP_TO_EDGE);
 321   gles2_if_->TexParameterf(
 322       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
 323       GL_CLAMP_TO_EDGE);
 324   // Allocate texture.
 325   gles2_if_->TexImage2D(
 326       context_->pp_resource(), GL_TEXTURE_2D, 0,
 327       rgba ? GL_BGRA_EXT : GL_LUMINANCE,
 328       width, height, 0,
 329       rgba ? GL_BGRA_EXT : GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
 330   AssertNoGLError();
 331   return texture_id;
 332 }
 333
 334 void MediaStreamVideoDemoInstance::CreateGLObjects() {
 335   // Code and constants for shader.
 336   static const char kVertexShader[] =
 337       "varying vec2 v_texCoord;            \n"
 338       "attribute vec4 a_position;          \n"
 339       "attribute vec2 a_texCoord;          \n"
 340       "void main()                         \n"
 341       "{                                   \n"
 342       "    v_texCoord = a_texCoord;        \n"
 343       "    gl_Position = a_position;       \n"
 344       "}";
 345
 346   static const char kFragmentShaderYUV[] =
 347       "precision mediump float;                                   \n"
 348       "varying vec2 v_texCoord;                                   \n"
 349       "uniform sampler2D y_texture;                               \n"
 350       "uniform sampler2D u_texture;                               \n"
 351       "uniform sampler2D v_texture;                               \n"
 352       "uniform mat3 color_matrix;                                 \n"
 353       "void main()                                                \n"
 354       "{                                                          \n"
 355       "  vec3 yuv;                                                \n"
 356       "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
 357       "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
 358       "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
 359       "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
 360       "  gl_FragColor = vec4(rgb, 1.0);                           \n"
 361       "}";
 362
 363   static const char kFragmentShaderRGB[] =
 364       "precision mediump float;                                   \n"
 365       "varying vec2 v_texCoord;                                   \n"
 366       "uniform sampler2D rgb_texture;                             \n"
 367       "void main()                                                \n"
 368       "{                                                          \n"
 369       "  gl_FragColor = texture2D(rgb_texture, v_texCoord);       \n"
 370       "}";
 371
 372   PP_Resource context = context_->pp_resource();
 373
 374   // Create shader programs.
 375   program_yuv_ = gles2_if_->CreateProgram(context);
 376   CreateShader(program_yuv_, GL_VERTEX_SHADER,
 377                kVertexShader, sizeof(kVertexShader));
 378   CreateShader(program_yuv_, GL_FRAGMENT_SHADER,
 379                kFragmentShaderYUV, sizeof(kFragmentShaderYUV));
 380   gles2_if_->LinkProgram(context, program_yuv_);
 381   AssertNoGLError();
 382
 383   program_rgb_ = gles2_if_->CreateProgram(context);
 384   CreateShader(program_rgb_, GL_VERTEX_SHADER,
 385                kVertexShader, sizeof(kVertexShader));
 386   CreateShader(program_rgb_, GL_FRAGMENT_SHADER,
 387                kFragmentShaderRGB, sizeof(kFragmentShaderRGB));
 388   gles2_if_->LinkProgram(context, program_rgb_);
 389   AssertNoGLError();
 390
 391   // Assign vertex positions and texture coordinates to buffers for use in
 392   // shader program.
 393   static const float kVertices[] = {
 394     -1, 1, -1, -1, 0, 1, 0, -1,  // Position coordinates.
 395     0, 1, 0, -1, 1, 1, 1, -1,  // Position coordinates.
 396     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
 397     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
 398   };
 399
 400   gles2_if_->GenBuffers(context, 1, &buffer_);
 401   gles2_if_->BindBuffer(context, GL_ARRAY_BUFFER, buffer_);
 402   gles2_if_->BufferData(context, GL_ARRAY_BUFFER,
 403                         sizeof(kVertices), kVertices, GL_STATIC_DRAW);
 404   AssertNoGLError();
 405 }
 406
 407 void MediaStreamVideoDemoInstance::CreateShader(
 408     GLuint program, GLenum type, const char* source, int size) {
 409   PP_Resource context = context_->pp_resource();
 410   GLuint shader = gles2_if_->CreateShader(context, type);
 411   gles2_if_->ShaderSource(context, shader, 1, &source, &size);
 412   gles2_if_->CompileShader(context, shader);
 413   gles2_if_->AttachShader(context, program, shader);
 414   gles2_if_->DeleteShader(context, shader);
 415 }
 416
 417 void MediaStreamVideoDemoInstance::CreateTextures() {
 418   int32_t width = frame_size_.width();
 419   int32_t height = frame_size_.height();
 420   if (width == 0 || height == 0)
 421     return;
 422   if (texture_y_)
 423     gles2_if_->DeleteTextures(context_->pp_resource(), 1, &texture_y_);
 424   if (texture_u_)
 425     gles2_if_->DeleteTextures(context_->pp_resource(), 1, &texture_u_);
 426   if (texture_v_)
 427     gles2_if_->DeleteTextures(context_->pp_resource(), 1, &texture_v_);
 428   if (texture_rgb_)
 429     gles2_if_->DeleteTextures(context_->pp_resource(), 1, &texture_rgb_);
 430   texture_y_ = CreateTexture(width, height, 0, false);
 431
 432   texture_u_ = CreateTexture(width / 2, height / 2, 1, false);
 433   texture_v_ = CreateTexture(width / 2, height / 2, 2, false);
 434   texture_rgb_ = CreateTexture(width, height, 3, true);
 435 }
 436
 437 void MediaStreamVideoDemoInstance::ConfigureTrack() {
 438   const int32_t attrib_list[] = {
 439       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT, attrib_format_,
 440       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH, attrib_width_,
 441       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT, attrib_height_,
 442       PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE
 443     };
 444   video_track_.Configure(attrib_list, callback_factory_.NewCallback(
 445         &MediaStreamVideoDemoInstance::OnConfigure));
 446 }
 447
 448 void MediaStreamVideoDemoInstance::OnConfigure(int32_t result) {
 449   video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 450       &MediaStreamVideoDemoInstance::OnGetFrame));
 451 }
 452
 453 void MediaStreamVideoDemoInstance::OnGetFrame(
 454     int32_t result, pp::VideoFrame frame) {
 455   if (result != PP_OK)
 456     return;
 457   const char* data = static_cast<const char*>(frame.GetDataBuffer());
 458   pp::Size size;
 459   frame.GetSize(&size);
 460
 461   if (size != frame_size_) {
 462     frame_size_ = size;
 463     CreateTextures();
 464   }
 465
 466   is_bgra_ = (frame.GetFormat() == PP_VIDEOFRAME_FORMAT_BGRA);
 467
 468   int32_t width = frame_size_.width();
 469   int32_t height = frame_size_.height();
 470   if (!is_bgra_) {
 471     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0);
 472     gles2_if_->TexSubImage2D(
 473         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
 474         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 475
 476     data += width * height;
 477     width /= 2;
 478     height /= 2;
 479
 480     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE1);
 481     gles2_if_->TexSubImage2D(
 482         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
 483         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 484
 485     data += width * height;
 486     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE2);
 487     gles2_if_->TexSubImage2D(
 488         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
 489         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 490   } else {
 491     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE3);
 492     gles2_if_->TexSubImage2D(
 493         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
 494         GL_BGRA_EXT, GL_UNSIGNED_BYTE, data);
 495   }
 496
 497   if (is_painting_)
 498     needs_paint_ = true;
 499   else
 500     Render();
 501
 502   video_track_.RecycleFrame(frame);
 503   if (need_config_) {
 504     ConfigureTrack();
 505     need_config_ = false;
 506   } else {
 507     video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 508         &MediaStreamVideoDemoInstance::OnGetFrame));
 509   }
 510 }
 511
 512 pp::Instance* MediaStreamVideoModule::CreateInstance(PP_Instance instance) {
 513   return new MediaStreamVideoDemoInstance(instance, this);
 514 }
 515
 516 }  // anonymous namespace
 517
 518 namespace pp {
 519 // Factory function for your specialization of the Module object.
 520 Module* CreateModule() {
 521   return new MediaStreamVideoModule();
 522 }
 523 }  // namespace pp