ppapi/examples/video_capture/video_capture.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <stdlib.h>
   6 #include <string.h>
   7
   8 #include <map>
   9 #include <vector>
  10
  11 #include "ppapi/c/dev/ppb_video_capture_dev.h"
  12 #include "ppapi/c/pp_errors.h"
  13 #include "ppapi/c/ppb_opengles2.h"
  14 #include "ppapi/cpp/dev/buffer_dev.h"
  15 #include "ppapi/cpp/dev/device_ref_dev.h"
  16 #include "ppapi/cpp/dev/video_capture_dev.h"
  17 #include "ppapi/cpp/dev/video_capture_client_dev.h"
  18 #include "ppapi/cpp/completion_callback.h"
  19 #include "ppapi/cpp/graphics_3d_client.h"
  20 #include "ppapi/cpp/graphics_3d.h"
  21 #include "ppapi/cpp/instance.h"
  22 #include "ppapi/cpp/module.h"
  23 #include "ppapi/cpp/rect.h"
  24 #include "ppapi/cpp/var.h"
  25 #include "ppapi/lib/gl/include/GLES2/gl2.h"
  26 #include "ppapi/utility/completion_callback_factory.h"
  27
  28 // When compiling natively on Windows, PostMessage can be #define-d to
  29 // something else.
  30 #ifdef PostMessage
  31 #undef PostMessage
  32 #endif
  33
  34 // Assert |context_| isn't holding any GL Errors.  Done as a macro instead of a
  35 // function to preserve line number information in the failure message.
  36 #define AssertNoGLError() \
  37   PP_DCHECK(!gles2_if_->GetError(context_->pp_resource()));
  38
  39 namespace {
  40
  41 const char* const kDelimiter = "#__#";
  42
  43 // This object is the global object representing this plugin library as long
  44 // as it is loaded.
  45 class VCDemoModule : public pp::Module {
  46  public:
  47   VCDemoModule() : pp::Module() {}
  48   virtual ~VCDemoModule() {}
  49
  50   virtual pp::Instance* CreateInstance(PP_Instance instance);
  51 };
  52
  53 class VCDemoInstance : public pp::Instance,
  54                        public pp::Graphics3DClient,
  55                        public pp::VideoCaptureClient_Dev {
  56  public:
  57   VCDemoInstance(PP_Instance instance, pp::Module* module);
  58   virtual ~VCDemoInstance();
  59
  60   // pp::Instance implementation (see PPP_Instance).
  61   virtual void DidChangeView(const pp::Rect& position,
  62                              const pp::Rect& clip_ignored);
  63   virtual void HandleMessage(const pp::Var& message_data);
  64
  65   // pp::Graphics3DClient implementation.
  66   virtual void Graphics3DContextLost() {
  67     InitGL();
  68     CreateYUVTextures();
  69     Render();
  70   }
  71
  72   virtual void OnDeviceInfo(PP_Resource resource,
  73                             const PP_VideoCaptureDeviceInfo_Dev& info,
  74                             const std::vector<pp::Buffer_Dev>& buffers) {
  75     capture_info_ = info;
  76     buffers_ = buffers;
  77     CreateYUVTextures();
  78   }
  79
  80   virtual void OnStatus(PP_Resource resource, uint32_t status) {
  81   }
  82
  83   virtual void OnError(PP_Resource resource, uint32_t error) {
  84   }
  85
  86   virtual void OnBufferReady(PP_Resource resource, uint32_t buffer) {
  87     const char* data = static_cast<const char*>(buffers_[buffer].data());
  88     int32_t width = capture_info_.width;
  89     int32_t height = capture_info_.height;
  90     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0);
  91     gles2_if_->TexSubImage2D(
  92         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
  93         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
  94
  95     data += width * height;
  96     width /= 2;
  97     height /= 2;
  98
  99     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE1);
 100     gles2_if_->TexSubImage2D(
 101         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
 102         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 103
 104     data += width * height;
 105     gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE2);
 106     gles2_if_->TexSubImage2D(
 107         context_->pp_resource(), GL_TEXTURE_2D, 0, 0, 0, width, height,
 108         GL_LUMINANCE, GL_UNSIGNED_BYTE, data);
 109
 110     video_capture_.ReuseBuffer(buffer);
 111     if (is_painting_)
 112       needs_paint_ = true;
 113     else
 114       Render();
 115   }
 116
 117  private:
 118   void Render();
 119
 120   // GL-related functions.
 121   void InitGL();
 122   GLuint CreateTexture(int32_t width, int32_t height, int unit);
 123   void CreateGLObjects();
 124   void CreateShader(GLuint program, GLenum type, const char* source, int size);
 125   void PaintFinished(int32_t result);
 126   void CreateYUVTextures();
 127
 128   void Open(const pp::DeviceRef_Dev& device);
 129   void Stop();
 130   void Start();
 131   void EnumerateDevicesFinished(int32_t result,
 132                                 std::vector<pp::DeviceRef_Dev>& devices);
 133   void OpenFinished(int32_t result);
 134
 135   static void MonitorDeviceChangeCallback(void* user_data,
 136                                           uint32_t device_count,
 137                                           const PP_Resource devices[]);
 138
 139   pp::Size position_size_;
 140   bool is_painting_;
 141   bool needs_paint_;
 142   GLuint texture_y_;
 143   GLuint texture_u_;
 144   GLuint texture_v_;
 145   pp::VideoCapture_Dev video_capture_;
 146   PP_VideoCaptureDeviceInfo_Dev capture_info_;
 147   std::vector<pp::Buffer_Dev> buffers_;
 148   pp::CompletionCallbackFactory<VCDemoInstance> callback_factory_;
 149
 150   // Unowned pointers.
 151   const struct PPB_OpenGLES2* gles2_if_;
 152
 153   // Owned data.
 154   pp::Graphics3D* context_;
 155
 156   std::vector<pp::DeviceRef_Dev> enumerate_devices_;
 157   std::vector<pp::DeviceRef_Dev> monitor_devices_;
 158 };
 159
 160 VCDemoInstance::VCDemoInstance(PP_Instance instance, pp::Module* module)
 161     : pp::Instance(instance),
 162       pp::Graphics3DClient(this),
 163       pp::VideoCaptureClient_Dev(this),
 164       is_painting_(false),
 165       needs_paint_(false),
 166       texture_y_(0),
 167       texture_u_(0),
 168       texture_v_(0),
 169       video_capture_(this),
 170       callback_factory_(this),
 171       context_(NULL) {
 172   gles2_if_ = static_cast<const struct PPB_OpenGLES2*>(
 173       module->GetBrowserInterface(PPB_OPENGLES2_INTERFACE));
 174   PP_DCHECK(gles2_if_);
 175
 176   capture_info_.width = 320;
 177   capture_info_.height = 240;
 178   capture_info_.frames_per_second = 30;
 179 }
 180
 181 VCDemoInstance::~VCDemoInstance() {
 182   video_capture_.MonitorDeviceChange(NULL, NULL);
 183   delete context_;
 184 }
 185
 186 void VCDemoInstance::DidChangeView(
 187     const pp::Rect& position, const pp::Rect& clip_ignored) {
 188   if (position.width() == 0 || position.height() == 0)
 189     return;
 190   if (position.size() == position_size_)
 191     return;
 192
 193   position_size_ = position.size();
 194
 195   // Initialize graphics.
 196   InitGL();
 197
 198   Render();
 199 }
 200
 201 void VCDemoInstance::HandleMessage(const pp::Var& message_data) {
 202   if (message_data.is_string()) {
 203     std::string event = message_data.AsString();
 204     if (event == "PageInitialized") {
 205       int32_t result = video_capture_.MonitorDeviceChange(
 206           &VCDemoInstance::MonitorDeviceChangeCallback, this);
 207       if (result != PP_OK)
 208         PostMessage(pp::Var("MonitorDeviceChangeFailed"));
 209
 210       pp::CompletionCallbackWithOutput<std::vector<pp::DeviceRef_Dev> >
 211           callback = callback_factory_.NewCallbackWithOutput(
 212               &VCDemoInstance::EnumerateDevicesFinished);
 213       result = video_capture_.EnumerateDevices(callback);
 214       if (result != PP_OK_COMPLETIONPENDING)
 215         PostMessage(pp::Var("EnumerationFailed"));
 216     } else if (event == "UseDefault") {
 217       Open(pp::DeviceRef_Dev());
 218     } else if (event == "Stop") {
 219       Stop();
 220     } else if (event == "Start") {
 221       Start();
 222     } else if (event.find("Monitor:") == 0) {
 223       std::string index_str = event.substr(strlen("Monitor:"));
 224       int index = atoi(index_str.c_str());
 225       if (index >= 0 && index < static_cast<int>(monitor_devices_.size()))
 226         Open(monitor_devices_[index]);
 227       else
 228         PP_NOTREACHED();
 229     } else if (event.find("Enumerate:") == 0) {
 230       std::string index_str = event.substr(strlen("Enumerate:"));
 231       int index = atoi(index_str.c_str());
 232       if (index >= 0 && index < static_cast<int>(enumerate_devices_.size()))
 233         Open(enumerate_devices_[index]);
 234       else
 235         PP_NOTREACHED();
 236     }
 237   }
 238 }
 239
 240 void VCDemoInstance::InitGL() {
 241   PP_DCHECK(position_size_.width() && position_size_.height());
 242   is_painting_ = false;
 243
 244   delete context_;
 245   int32_t attributes[] = {
 246     PP_GRAPHICS3DATTRIB_ALPHA_SIZE, 0,
 247     PP_GRAPHICS3DATTRIB_BLUE_SIZE, 8,
 248     PP_GRAPHICS3DATTRIB_GREEN_SIZE, 8,
 249     PP_GRAPHICS3DATTRIB_RED_SIZE, 8,
 250     PP_GRAPHICS3DATTRIB_DEPTH_SIZE, 0,
 251     PP_GRAPHICS3DATTRIB_STENCIL_SIZE, 0,
 252     PP_GRAPHICS3DATTRIB_SAMPLES, 0,
 253     PP_GRAPHICS3DATTRIB_SAMPLE_BUFFERS, 0,
 254     PP_GRAPHICS3DATTRIB_WIDTH, position_size_.width(),
 255     PP_GRAPHICS3DATTRIB_HEIGHT, position_size_.height(),
 256     PP_GRAPHICS3DATTRIB_NONE,
 257   };
 258   context_ = new pp::Graphics3D(this, attributes);
 259   PP_DCHECK(!context_->is_null());
 260
 261   // Set viewport window size and clear color bit.
 262   gles2_if_->ClearColor(context_->pp_resource(), 1, 0, 0, 1);
 263   gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
 264   gles2_if_->Viewport(context_->pp_resource(), 0, 0,
 265                       position_size_.width(), position_size_.height());
 266
 267   BindGraphics(*context_);
 268   AssertNoGLError();
 269
 270   CreateGLObjects();
 271 }
 272
 273 void VCDemoInstance::Render() {
 274   PP_DCHECK(!is_painting_);
 275   is_painting_ = true;
 276   needs_paint_ = false;
 277   if (texture_y_) {
 278     gles2_if_->DrawArrays(context_->pp_resource(), GL_TRIANGLE_STRIP, 0, 4);
 279   } else {
 280     gles2_if_->Clear(context_->pp_resource(), GL_COLOR_BUFFER_BIT);
 281   }
 282   pp::CompletionCallback cb = callback_factory_.NewCallback(
 283       &VCDemoInstance::PaintFinished);
 284   context_->SwapBuffers(cb);
 285 }
 286
 287 void VCDemoInstance::PaintFinished(int32_t result) {
 288   is_painting_ = false;
 289   if (needs_paint_)
 290     Render();
 291 }
 292
 293 GLuint VCDemoInstance::CreateTexture(int32_t width, int32_t height, int unit) {
 294   GLuint texture_id;
 295   gles2_if_->GenTextures(context_->pp_resource(), 1, &texture_id);
 296   AssertNoGLError();
 297   // Assign parameters.
 298   gles2_if_->ActiveTexture(context_->pp_resource(), GL_TEXTURE0 + unit);
 299   gles2_if_->BindTexture(context_->pp_resource(), GL_TEXTURE_2D, texture_id);
 300   gles2_if_->TexParameteri(
 301       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
 302       GL_NEAREST);
 303   gles2_if_->TexParameteri(
 304       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
 305       GL_NEAREST);
 306   gles2_if_->TexParameterf(
 307       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
 308       GL_CLAMP_TO_EDGE);
 309   gles2_if_->TexParameterf(
 310       context_->pp_resource(), GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
 311       GL_CLAMP_TO_EDGE);
 312
 313   // Allocate texture.
 314   gles2_if_->TexImage2D(
 315       context_->pp_resource(), GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0,
 316       GL_LUMINANCE, GL_UNSIGNED_BYTE, NULL);
 317   AssertNoGLError();
 318   return texture_id;
 319 }
 320
 321 void VCDemoInstance::CreateGLObjects() {
 322   // Code and constants for shader.
 323   static const char kVertexShader[] =
 324       "varying vec2 v_texCoord;            \n"
 325       "attribute vec4 a_position;          \n"
 326       "attribute vec2 a_texCoord;          \n"
 327       "void main()                         \n"
 328       "{                                   \n"
 329       "    v_texCoord = a_texCoord;        \n"
 330       "    gl_Position = a_position;       \n"
 331       "}";
 332
 333   static const char kFragmentShader[] =
 334       "precision mediump float;                                   \n"
 335       "varying vec2 v_texCoord;                                   \n"
 336       "uniform sampler2D y_texture;                               \n"
 337       "uniform sampler2D u_texture;                               \n"
 338       "uniform sampler2D v_texture;                               \n"
 339       "uniform mat3 color_matrix;                                 \n"
 340       "void main()                                                \n"
 341       "{                                                          \n"
 342       "  vec3 yuv;                                                \n"
 343       "  yuv.x = texture2D(y_texture, v_texCoord).r;              \n"
 344       "  yuv.y = texture2D(u_texture, v_texCoord).r;              \n"
 345       "  yuv.z = texture2D(v_texture, v_texCoord).r;              \n"
 346       "  vec3 rgb = color_matrix * (yuv - vec3(0.0625, 0.5, 0.5));\n"
 347       "  gl_FragColor = vec4(rgb, 1.0);                           \n"
 348       "}";
 349
 350   static const float kColorMatrix[9] = {
 351     1.1643828125f, 1.1643828125f, 1.1643828125f,
 352     0.0f, -0.39176171875f, 2.017234375f,
 353     1.59602734375f, -0.81296875f, 0.0f
 354   };
 355
 356   PP_Resource context = context_->pp_resource();
 357
 358   // Create shader program.
 359   GLuint program = gles2_if_->CreateProgram(context);
 360   CreateShader(program, GL_VERTEX_SHADER, kVertexShader, sizeof(kVertexShader));
 361   CreateShader(
 362       program, GL_FRAGMENT_SHADER, kFragmentShader, sizeof(kFragmentShader));
 363   gles2_if_->LinkProgram(context, program);
 364   gles2_if_->UseProgram(context, program);
 365   gles2_if_->DeleteProgram(context, program);
 366   gles2_if_->Uniform1i(
 367       context, gles2_if_->GetUniformLocation(context, program, "y_texture"), 0);
 368   gles2_if_->Uniform1i(
 369       context, gles2_if_->GetUniformLocation(context, program, "u_texture"), 1);
 370   gles2_if_->Uniform1i(
 371       context, gles2_if_->GetUniformLocation(context, program, "v_texture"), 2);
 372   gles2_if_->UniformMatrix3fv(
 373       context,
 374       gles2_if_->GetUniformLocation(context, program, "color_matrix"),
 375       1, GL_FALSE, kColorMatrix);
 376   AssertNoGLError();
 377
 378   // Assign vertex positions and texture coordinates to buffers for use in
 379   // shader program.
 380   static const float kVertices[] = {
 381     -1, 1, -1, -1, 1, 1, 1, -1,  // Position coordinates.
 382     0, 0, 0, 1, 1, 0, 1, 1,  // Texture coordinates.
 383   };
 384
 385   GLuint buffer;
 386   gles2_if_->GenBuffers(context, 1, &buffer);
 387   gles2_if_->BindBuffer(context, GL_ARRAY_BUFFER, buffer);
 388   gles2_if_->BufferData(context, GL_ARRAY_BUFFER,
 389                         sizeof(kVertices), kVertices, GL_STATIC_DRAW);
 390   AssertNoGLError();
 391   GLint pos_location = gles2_if_->GetAttribLocation(
 392       context, program, "a_position");
 393   GLint tc_location = gles2_if_->GetAttribLocation(
 394       context, program, "a_texCoord");
 395   AssertNoGLError();
 396   gles2_if_->EnableVertexAttribArray(context, pos_location);
 397   gles2_if_->VertexAttribPointer(context, pos_location, 2,
 398                                  GL_FLOAT, GL_FALSE, 0, 0);
 399   gles2_if_->EnableVertexAttribArray(context, tc_location);
 400   gles2_if_->VertexAttribPointer(
 401       context, tc_location, 2, GL_FLOAT, GL_FALSE, 0,
 402       static_cast<float*>(0) + 8);  // Skip position coordinates.
 403   AssertNoGLError();
 404 }
 405
 406 void VCDemoInstance::CreateShader(
 407     GLuint program, GLenum type, const char* source, int size) {
 408   PP_Resource context = context_->pp_resource();
 409   GLuint shader = gles2_if_->CreateShader(context, type);
 410   gles2_if_->ShaderSource(context, shader, 1, &source, &size);
 411   gles2_if_->CompileShader(context, shader);
 412   gles2_if_->AttachShader(context, program, shader);
 413   gles2_if_->DeleteShader(context, shader);
 414 }
 415
 416 void VCDemoInstance::CreateYUVTextures() {
 417   int32_t width = capture_info_.width;
 418   int32_t height = capture_info_.height;
 419   texture_y_ = CreateTexture(width, height, 0);
 420
 421   width /= 2;
 422   height /= 2;
 423   texture_u_ = CreateTexture(width, height, 1);
 424   texture_v_ = CreateTexture(width, height, 2);
 425 }
 426
 427 void VCDemoInstance::Open(const pp::DeviceRef_Dev& device) {
 428   pp::CompletionCallback callback = callback_factory_.NewCallback(
 429       &VCDemoInstance::OpenFinished);
 430   int32_t result = video_capture_.Open(device, capture_info_, 4, callback);
 431   if (result != PP_OK_COMPLETIONPENDING)
 432     PostMessage(pp::Var("OpenFailed"));
 433 }
 434
 435 void VCDemoInstance::Stop() {
 436   if (video_capture_.StopCapture() != PP_OK)
 437     PostMessage(pp::Var("StopFailed"));
 438 }
 439
 440 void VCDemoInstance::Start() {
 441   if (video_capture_.StartCapture() != PP_OK)
 442     PostMessage(pp::Var("StartFailed"));
 443 }
 444
 445 void VCDemoInstance::EnumerateDevicesFinished(
 446     int32_t result,
 447   std::vector<pp::DeviceRef_Dev>& devices) {
 448   if (result == PP_OK) {
 449     enumerate_devices_.swap(devices);
 450     std::string device_names = "Enumerate:";
 451     for (size_t index = 0; index < enumerate_devices_.size(); ++index) {
 452       pp::Var name = enumerate_devices_[index].GetName();
 453       PP_DCHECK(name.is_string());
 454
 455       if (index != 0)
 456         device_names += kDelimiter;
 457       device_names += name.AsString();
 458     }
 459     PostMessage(pp::Var(device_names));
 460   } else {
 461     PostMessage(pp::Var("EnumerationFailed"));
 462   }
 463 }
 464
 465 void VCDemoInstance::OpenFinished(int32_t result) {
 466   if (result == PP_OK)
 467     Start();
 468   else
 469     PostMessage(pp::Var("OpenFailed"));
 470 }
 471
 472 // static
 473 void VCDemoInstance::MonitorDeviceChangeCallback(void* user_data,
 474                                                  uint32_t device_count,
 475                                                  const PP_Resource devices[]) {
 476   VCDemoInstance* thiz = static_cast<VCDemoInstance*>(user_data);
 477
 478   std::string device_names = "Monitor:";
 479   thiz->monitor_devices_.clear();
 480   thiz->monitor_devices_.reserve(device_count);
 481   for (size_t index = 0; index < device_count; ++index) {
 482     thiz->monitor_devices_.push_back(pp::DeviceRef_Dev(devices[index]));
 483     pp::Var name = thiz->monitor_devices_.back().GetName();
 484     PP_DCHECK(name.is_string());
 485
 486     if (index != 0)
 487       device_names += kDelimiter;
 488     device_names += name.AsString();
 489   }
 490   thiz->PostMessage(pp::Var(device_names));
 491 }
 492
 493 pp::Instance* VCDemoModule::CreateInstance(PP_Instance instance) {
 494   return new VCDemoInstance(instance, this);
 495 }
 496
 497 }  // anonymous namespace
 498
 499 namespace pp {
 500 // Factory function for your specialization of the Module object.
 501 Module* CreateModule() {
 502   return new VCDemoModule();
 503 }
 504 }  // namespace pp