native_client_sdk/src/examples/api/video_encode/video_encode.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6 #include <stdio.h>
   7 #include <string.h>
   8
   9 #include <iostream>
  10 #include <map>
  11 #include <sstream>
  12 #include <vector>
  13
  14 #include "ppapi/c/pp_errors.h"
  15 #include "ppapi/c/ppb_console.h"
  16 #include "ppapi/cpp/input_event.h"
  17 #include "ppapi/cpp/instance.h"
  18 #include "ppapi/cpp/media_stream_video_track.h"
  19 #include "ppapi/cpp/module.h"
  20 #include "ppapi/cpp/rect.h"
  21 #include "ppapi/cpp/var.h"
  22 #include "ppapi/cpp/var_array_buffer.h"
  23 #include "ppapi/cpp/var_dictionary.h"
  24 #include "ppapi/cpp/video_encoder.h"
  25 #include "ppapi/cpp/video_frame.h"
  26 #include "ppapi/utility/completion_callback_factory.h"
  27
  28 // When compiling natively on Windows, PostMessage can be #define-d to
  29 // something else.
  30 #ifdef PostMessage
  31 #undef PostMessage
  32 #endif
  33
  34 // Use assert as a poor-man's CHECK, even in non-debug mode.
  35 // Since <assert.h> redefines assert on every inclusion (it doesn't use
  36 // include-guards), make sure this is the last file #include'd in this file.
  37 #undef NDEBUG
  38 #include <assert.h>
  39
  40 #define fourcc(a, b, c, d)                                               \
  41   (((uint32_t)(a) << 0) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | \
  42    ((uint32_t)(d) << 24))
  43
  44 namespace {
  45
  46 // IVF container writer. It is possible to parse H264 bitstream using
  47 // NAL units but for VP8 we need a container to at least find encoded
  48 // pictures as well as the picture sizes.
  49 class IVFWriter {
  50  public:
  51   IVFWriter() {}
  52   ~IVFWriter() {}
  53
  54   uint32_t GetFileHeaderSize() const { return 32; }
  55   uint32_t GetFrameHeaderSize() const { return 12; }
  56   uint32_t WriteFileHeader(uint8_t* mem, int32_t width, int32_t height);
  57   uint32_t WriteFrameHeader(uint8_t* mem, uint64_t pts, size_t frame_size);
  58
  59  private:
  60   void PutLE16(uint8_t* mem, int val) const {
  61     mem[0] = (val >> 0) & 0xff;
  62     mem[1] = (val >> 8) & 0xff;
  63   }
  64   void PutLE32(uint8_t* mem, int val) const {
  65     mem[0] = (val >> 0) & 0xff;
  66     mem[1] = (val >> 8) & 0xff;
  67     mem[2] = (val >> 16) & 0xff;
  68     mem[3] = (val >> 24) & 0xff;
  69   }
  70 };
  71
  72 uint32_t IVFWriter::WriteFileHeader(uint8_t* mem,
  73                                     int32_t width,
  74                                     int32_t height) {
  75   mem[0] = 'D';
  76   mem[1] = 'K';
  77   mem[2] = 'I';
  78   mem[3] = 'F';
  79   PutLE16(mem + 4, 0);                               // version
  80   PutLE16(mem + 6, 32);                              // header size
  81   PutLE32(mem + 8, fourcc('V', 'P', '8', '0'));      // fourcc
  82   PutLE16(mem + 12, static_cast<uint16_t>(width));   // width
  83   PutLE16(mem + 14, static_cast<uint16_t>(height));  // height
  84   PutLE32(mem + 16, 30);                             // rate
  85   PutLE32(mem + 20, 1);                              // scale
  86   PutLE32(mem + 24, 0xffffffff);                     // length
  87   PutLE32(mem + 28, 0);                              // unused
  88
  89   return 32;
  90 }
  91
  92 uint32_t IVFWriter::WriteFrameHeader(uint8_t* mem,
  93                                      uint64_t pts,
  94                                      size_t frame_size) {
  95   PutLE32(mem, (int)frame_size);
  96   PutLE32(mem + 4, (int)(pts & 0xFFFFFFFF));
  97   PutLE32(mem + 8, (int)(pts >> 32));
  98
  99   return 12;
 100 }
 101
 102 // This object is the global object representing this plugin library as long
 103 // as it is loaded.
 104 class VideoEncoderModule : public pp::Module {
 105  public:
 106   VideoEncoderModule() : pp::Module() {}
 107   virtual ~VideoEncoderModule() {}
 108
 109   virtual pp::Instance* CreateInstance(PP_Instance instance);
 110 };
 111
 112 class VideoEncoderInstance : public pp::Instance {
 113  public:
 114   VideoEncoderInstance(PP_Instance instance, pp::Module* module);
 115   virtual ~VideoEncoderInstance();
 116
 117   // pp::Instance implementation.
 118   virtual void HandleMessage(const pp::Var& var_message);
 119
 120  private:
 121   void AddVideoProfile(PP_VideoProfile profile, const std::string& profile_str);
 122   void InitializeVideoProfiles();
 123   PP_VideoProfile VideoProfileFromString(const std::string& str);
 124   std::string VideoProfileToString(PP_VideoProfile profile);
 125
 126   void ConfigureTrack();
 127   void OnConfiguredTrack(int32_t result);
 128   void ProbeEncoder();
 129   void OnEncoderProbed(int32_t result,
 130                        const std::vector<PP_VideoProfileDescription> profiles);
 131   void StartEncoder();
 132   void OnInitializedEncoder(int32_t result);
 133   void ScheduleNextEncode();
 134   void GetEncoderFrameTick(int32_t result);
 135   void GetEncoderFrame(const pp::VideoFrame& track_frame);
 136   void OnEncoderFrame(int32_t result,
 137                       pp::VideoFrame encoder_frame,
 138                       pp::VideoFrame track_frame);
 139   int32_t CopyVideoFrame(pp::VideoFrame dest, pp::VideoFrame src);
 140   void EncodeFrame(const pp::VideoFrame& frame);
 141   void OnEncodeDone(int32_t result);
 142   void OnGetBitstreamBuffer(int32_t result, PP_BitstreamBuffer buffer);
 143   void StartTrackFrames();
 144   void StopTrackFrames();
 145   void OnTrackFrame(int32_t result, pp::VideoFrame frame);
 146
 147   void StopEncode();
 148
 149   void LogError(int32_t error, const std::string& message);
 150   void Log(const std::string& message);
 151
 152   void PostDataMessage(const void* buffer, uint32_t size);
 153
 154   typedef std::map<std::string, PP_VideoProfile> VideoProfileFromStringMap;
 155   VideoProfileFromStringMap profile_from_string_;
 156
 157   typedef std::map<PP_VideoProfile, std::string> VideoProfileToStringMap;
 158   VideoProfileToStringMap profile_to_string_;
 159
 160   bool is_encoding_;
 161   bool is_receiving_track_frames_;
 162
 163   pp::VideoEncoder video_encoder_;
 164   pp::MediaStreamVideoTrack video_track_;
 165   pp::CompletionCallbackFactory<VideoEncoderInstance> callback_factory_;
 166
 167   PP_VideoProfile video_profile_;
 168   PP_VideoFrame_Format frame_format_;
 169
 170   pp::Size requested_size_;
 171   pp::Size frame_size_;
 172   pp::Size encoder_size_;
 173   uint32_t encoded_frames_;
 174
 175   pp::VideoFrame current_track_frame_;
 176
 177   IVFWriter ivf_writer_;
 178 };
 179
 180 VideoEncoderInstance::VideoEncoderInstance(PP_Instance instance,
 181                                            pp::Module* module)
 182     : pp::Instance(instance),
 183       is_encoding_(false),
 184       callback_factory_(this),
 185 #if defined(USE_VP8_INSTEAD_OF_H264)
 186       video_profile_(PP_VIDEOPROFILE_VP8_ANY),
 187 #else
 188       video_profile_(PP_VIDEOPROFILE_H264MAIN),
 189 #endif
 190       frame_format_(PP_VIDEOFRAME_FORMAT_I420),
 191       encoded_frames_(0) {
 192   InitializeVideoProfiles();
 193   ProbeEncoder();
 194 }
 195
 196 VideoEncoderInstance::~VideoEncoderInstance() {
 197 }
 198
 199 void VideoEncoderInstance::AddVideoProfile(PP_VideoProfile profile,
 200                                            const std::string& profile_str) {
 201   profile_to_string_.insert(std::make_pair(profile, profile_str));
 202   profile_from_string_.insert(std::make_pair(profile_str, profile));
 203 }
 204
 205 void VideoEncoderInstance::InitializeVideoProfiles() {
 206   AddVideoProfile(PP_VIDEOPROFILE_H264BASELINE, "h264baseline");
 207   AddVideoProfile(PP_VIDEOPROFILE_H264MAIN, "h264main");
 208   AddVideoProfile(PP_VIDEOPROFILE_H264EXTENDED, "h264extended");
 209   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH, "h264high");
 210   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH10PROFILE, "h264high10");
 211   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH422PROFILE, "h264high422");
 212   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH444PREDICTIVEPROFILE,
 213                   "h264high444predictive");
 214   AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEBASELINE, "h264scalablebaseline");
 215   AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEHIGH, "h264scalablehigh");
 216   AddVideoProfile(PP_VIDEOPROFILE_H264STEREOHIGH, "h264stereohigh");
 217   AddVideoProfile(PP_VIDEOPROFILE_H264MULTIVIEWHIGH, "h264multiviewhigh");
 218   AddVideoProfile(PP_VIDEOPROFILE_VP8_ANY, "vp8");
 219   AddVideoProfile(PP_VIDEOPROFILE_VP9_ANY, "vp9");
 220 }
 221
 222 PP_VideoProfile VideoEncoderInstance::VideoProfileFromString(
 223     const std::string& str) {
 224   VideoProfileFromStringMap::iterator it = profile_from_string_.find(str);
 225   if (it == profile_from_string_.end())
 226     return PP_VIDEOPROFILE_VP8_ANY;
 227   return it->second;
 228 }
 229
 230 std::string VideoEncoderInstance::VideoProfileToString(
 231     PP_VideoProfile profile) {
 232   VideoProfileToStringMap::iterator it = profile_to_string_.find(profile);
 233   if (it == profile_to_string_.end())
 234     return "unknown";
 235   return it->second;
 236 }
 237
 238 void VideoEncoderInstance::ConfigureTrack() {
 239   if (encoder_size_.IsEmpty())
 240     frame_size_ = requested_size_;
 241   else
 242     frame_size_ = encoder_size_;
 243
 244   int32_t attrib_list[] = {PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT,
 245                            frame_format_,
 246                            PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH,
 247                            frame_size_.width(),
 248                            PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT,
 249                            frame_size_.height(),
 250                            PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE};
 251
 252   video_track_.Configure(
 253       attrib_list,
 254       callback_factory_.NewCallback(&VideoEncoderInstance::OnConfiguredTrack));
 255 }
 256
 257 void VideoEncoderInstance::OnConfiguredTrack(int32_t result) {
 258   if (result != PP_OK) {
 259     LogError(result, "Cannot configure track");
 260     return;
 261   }
 262
 263   if (is_encoding_) {
 264     StartTrackFrames();
 265     ScheduleNextEncode();
 266   } else
 267     StartEncoder();
 268 }
 269
 270 void VideoEncoderInstance::ProbeEncoder() {
 271   video_encoder_ = pp::VideoEncoder(this);
 272   video_encoder_.GetSupportedProfiles(callback_factory_.NewCallbackWithOutput(
 273       &VideoEncoderInstance::OnEncoderProbed));
 274 }
 275
 276 void VideoEncoderInstance::OnEncoderProbed(
 277     int32_t result,
 278     const std::vector<PP_VideoProfileDescription> profiles) {
 279   pp::VarDictionary dict;
 280   dict.Set(pp::Var("name"), pp::Var("supportedProfiles"));
 281   pp::VarArray js_profiles;
 282   dict.Set(pp::Var("profiles"), js_profiles);
 283
 284   if (result < 0) {
 285     LogError(result, "Cannot get supported profiles");
 286     PostMessage(dict);
 287   }
 288
 289   int32_t idx = 0;
 290   for (uint32_t i = 0; i < profiles.size(); i++) {
 291     const PP_VideoProfileDescription& profile = profiles[i];
 292     js_profiles.Set(idx++, pp::Var(VideoProfileToString(profile.profile)));
 293   }
 294   PostMessage(dict);
 295 }
 296
 297 void VideoEncoderInstance::StartEncoder() {
 298   video_encoder_ = pp::VideoEncoder(this);
 299
 300   int32_t error = video_encoder_.Initialize(
 301       frame_format_, frame_size_, video_profile_, 2000000,
 302       PP_HARDWAREACCELERATION_WITHFALLBACK,
 303       callback_factory_.NewCallback(
 304           &VideoEncoderInstance::OnInitializedEncoder));
 305   if (error != PP_OK_COMPLETIONPENDING) {
 306     LogError(error, "Cannot initialize encoder");
 307     return;
 308   }
 309 }
 310
 311 void VideoEncoderInstance::OnInitializedEncoder(int32_t result) {
 312   if (result != PP_OK) {
 313     LogError(result, "Encoder initialization failed");
 314     return;
 315   }
 316
 317   is_encoding_ = true;
 318   Log("started");
 319
 320   if (video_encoder_.GetFrameCodedSize(&encoder_size_) != PP_OK) {
 321     LogError(result, "Cannot get encoder coded frame size");
 322     return;
 323   }
 324
 325   video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
 326       &VideoEncoderInstance::OnGetBitstreamBuffer));
 327
 328   if (encoder_size_ != frame_size_)
 329     ConfigureTrack();
 330   else {
 331     StartTrackFrames();
 332     ScheduleNextEncode();
 333   }
 334 }
 335
 336 void VideoEncoderInstance::ScheduleNextEncode() {
 337   pp::Module::Get()->core()->CallOnMainThread(
 338       1000 / 30,
 339       callback_factory_.NewCallback(&VideoEncoderInstance::GetEncoderFrameTick),
 340       0);
 341 }
 342
 343 void VideoEncoderInstance::GetEncoderFrameTick(int32_t result) {
 344   if (is_encoding_) {
 345     if (!current_track_frame_.is_null()) {
 346       pp::VideoFrame frame = current_track_frame_;
 347       current_track_frame_.detach();
 348       GetEncoderFrame(frame);
 349     }
 350     ScheduleNextEncode();
 351   }
 352 }
 353
 354 void VideoEncoderInstance::GetEncoderFrame(const pp::VideoFrame& track_frame) {
 355   video_encoder_.GetVideoFrame(callback_factory_.NewCallbackWithOutput(
 356       &VideoEncoderInstance::OnEncoderFrame, track_frame));
 357 }
 358
 359 void VideoEncoderInstance::OnEncoderFrame(int32_t result,
 360                                           pp::VideoFrame encoder_frame,
 361                                           pp::VideoFrame track_frame) {
 362   if (result == PP_ERROR_ABORTED) {
 363     video_track_.RecycleFrame(track_frame);
 364     return;
 365   }
 366   if (result != PP_OK) {
 367     video_track_.RecycleFrame(track_frame);
 368     LogError(result, "Cannot get video frame from video encoder");
 369     return;
 370   }
 371
 372   track_frame.GetSize(&frame_size_);
 373
 374   if (frame_size_ != encoder_size_) {
 375     video_track_.RecycleFrame(track_frame);
 376     LogError(PP_ERROR_FAILED, "MediaStreamVideoTrack frame size incorrect");
 377     return;
 378   }
 379
 380   if (CopyVideoFrame(encoder_frame, track_frame) == PP_OK)
 381     EncodeFrame(encoder_frame);
 382   video_track_.RecycleFrame(track_frame);
 383 }
 384
 385 int32_t VideoEncoderInstance::CopyVideoFrame(pp::VideoFrame dest,
 386                                              pp::VideoFrame src) {
 387   if (dest.GetDataBufferSize() < src.GetDataBufferSize()) {
 388     std::ostringstream oss;
 389     oss << "Incorrect destination video frame buffer size : "
 390         << dest.GetDataBufferSize() << " < " << src.GetDataBufferSize();
 391     LogError(PP_ERROR_FAILED, oss.str());
 392     return PP_ERROR_FAILED;
 393   }
 394
 395   dest.SetTimestamp(src.GetTimestamp());
 396   memcpy(dest.GetDataBuffer(), src.GetDataBuffer(), src.GetDataBufferSize());
 397   return PP_OK;
 398 }
 399
 400 void VideoEncoderInstance::EncodeFrame(const pp::VideoFrame& frame) {
 401   video_encoder_.Encode(
 402       frame, PP_FALSE,
 403       callback_factory_.NewCallback(&VideoEncoderInstance::OnEncodeDone));
 404 }
 405
 406 void VideoEncoderInstance::OnEncodeDone(int32_t result) {
 407   if (result == PP_ERROR_ABORTED)
 408     return;
 409   if (result != PP_OK)
 410     LogError(result, "Encode failed");
 411 }
 412
 413 void VideoEncoderInstance::OnGetBitstreamBuffer(int32_t result,
 414                                                 PP_BitstreamBuffer buffer) {
 415   if (result == PP_ERROR_ABORTED)
 416     return;
 417   if (result != PP_OK) {
 418     LogError(result, "Cannot get bitstream buffer");
 419     return;
 420   }
 421
 422   encoded_frames_++;
 423   PostDataMessage(buffer.buffer, buffer.size);
 424   video_encoder_.RecycleBitstreamBuffer(buffer);
 425
 426   video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
 427       &VideoEncoderInstance::OnGetBitstreamBuffer));
 428 }
 429
 430 void VideoEncoderInstance::StartTrackFrames() {
 431   is_receiving_track_frames_ = true;
 432   video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 433       &VideoEncoderInstance::OnTrackFrame));
 434 }
 435
 436 void VideoEncoderInstance::StopTrackFrames() {
 437   is_receiving_track_frames_ = false;
 438   if (!current_track_frame_.is_null()) {
 439     video_track_.RecycleFrame(current_track_frame_);
 440     current_track_frame_.detach();
 441   }
 442 }
 443
 444 void VideoEncoderInstance::OnTrackFrame(int32_t result, pp::VideoFrame frame) {
 445   if (result == PP_ERROR_ABORTED)
 446     return;
 447
 448   if (!current_track_frame_.is_null()) {
 449     video_track_.RecycleFrame(current_track_frame_);
 450     current_track_frame_.detach();
 451   }
 452
 453   if (result != PP_OK) {
 454     LogError(result, "Cannot get video frame from video track");
 455     return;
 456   }
 457
 458   current_track_frame_ = frame;
 459   if (is_receiving_track_frames_)
 460     video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 461         &VideoEncoderInstance::OnTrackFrame));
 462 }
 463
 464 void VideoEncoderInstance::StopEncode() {
 465   video_encoder_.Close();
 466   StopTrackFrames();
 467   video_track_.Close();
 468   is_encoding_ = false;
 469   encoded_frames_ = 0;
 470 }
 471
 472 //
 473
 474 void VideoEncoderInstance::HandleMessage(const pp::Var& var_message) {
 475   if (!var_message.is_dictionary()) {
 476     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
 477     return;
 478   }
 479
 480   pp::VarDictionary dict_message(var_message);
 481   std::string command = dict_message.Get("command").AsString();
 482
 483   if (command == "start") {
 484     requested_size_ = pp::Size(dict_message.Get("width").AsInt(),
 485                                dict_message.Get("height").AsInt());
 486     pp::Var var_track = dict_message.Get("track");
 487     if (!var_track.is_resource()) {
 488       LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Given track is not a resource"));
 489       return;
 490     }
 491     pp::Resource resource_track = var_track.AsResource();
 492     video_track_ = pp::MediaStreamVideoTrack(resource_track);
 493     video_encoder_ = pp::VideoEncoder();
 494     video_profile_ = VideoProfileFromString(
 495         dict_message.Get("profile").AsString());
 496     ConfigureTrack();
 497   } else if (command == "stop") {
 498     StopEncode();
 499     Log("stopped");
 500   } else {
 501     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
 502   }
 503 }
 504
 505 void VideoEncoderInstance::PostDataMessage(const void* buffer, uint32_t size) {
 506   pp::VarDictionary dictionary;
 507
 508   dictionary.Set(pp::Var("name"), pp::Var("data"));
 509
 510   pp::VarArrayBuffer array_buffer;
 511   uint8_t* data_ptr;
 512   uint32_t data_offset = 0;
 513   if (video_profile_ == PP_VIDEOPROFILE_VP8_ANY ||
 514       video_profile_ == PP_VIDEOPROFILE_VP9_ANY) {
 515     uint32_t frame_offset = 0;
 516     if (encoded_frames_ == 1) {
 517       array_buffer = pp::VarArrayBuffer(
 518           size + ivf_writer_.GetFileHeaderSize() +
 519           ivf_writer_.GetFrameHeaderSize());
 520       data_ptr = static_cast<uint8_t*>(array_buffer.Map());
 521       frame_offset = ivf_writer_.WriteFileHeader(
 522           data_ptr, frame_size_.width(), frame_size_.height());
 523     } else {
 524       array_buffer = pp::VarArrayBuffer(
 525           size + ivf_writer_.GetFrameHeaderSize());
 526       data_ptr = static_cast<uint8_t*>(array_buffer.Map());
 527     }
 528     data_offset = frame_offset +
 529       ivf_writer_.WriteFrameHeader(data_ptr + frame_offset,
 530                                    encoded_frames_,
 531                                    size);
 532   } else {
 533     array_buffer = pp::VarArrayBuffer(size);
 534     data_ptr = static_cast<uint8_t*>(array_buffer.Map());
 535   }
 536
 537   memcpy(data_ptr + data_offset, buffer, size);
 538   array_buffer.Unmap();
 539   dictionary.Set(pp::Var("data"), array_buffer);
 540
 541   PostMessage(dictionary);
 542 }
 543
 544 void VideoEncoderInstance::LogError(int32_t error, const std::string& message) {
 545   std::string msg("Error: ");
 546   msg.append(pp::Var(error).DebugString());
 547   msg.append(" : ");
 548   msg.append(message);
 549
 550   Log(msg);
 551 }
 552
 553 void VideoEncoderInstance::Log(const std::string& message) {
 554   pp::VarDictionary dictionary;
 555   dictionary.Set(pp::Var("name"), pp::Var("log"));
 556   dictionary.Set(pp::Var("message"), pp::Var(message));
 557
 558   PostMessage(dictionary);
 559 }
 560
 561 pp::Instance* VideoEncoderModule::CreateInstance(PP_Instance instance) {
 562   return new VideoEncoderInstance(instance, this);
 563 }
 564
 565 }  // anonymous namespace
 566
 567 namespace pp {
 568 // Factory function for your specialization of the Module object.
 569 Module* CreateModule() {
 570   return new VideoEncoderModule();
 571 }
 572 }  // namespace pp