ppapi/examples/video_encode/video_encode.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6 #include <stdio.h>
   7 #include <string.h>
   8
   9 #include <iostream>
  10 #include <map>
  11 #include <sstream>
  12 #include <vector>
  13
  14 #include "ppapi/c/pp_errors.h"
  15 #include "ppapi/c/ppb_console.h"
  16 #include "ppapi/cpp/input_event.h"
  17 #include "ppapi/cpp/instance.h"
  18 #include "ppapi/cpp/media_stream_video_track.h"
  19 #include "ppapi/cpp/module.h"
  20 #include "ppapi/cpp/rect.h"
  21 #include "ppapi/cpp/var.h"
  22 #include "ppapi/cpp/var_array_buffer.h"
  23 #include "ppapi/cpp/var_dictionary.h"
  24 #include "ppapi/cpp/video_encoder.h"
  25 #include "ppapi/cpp/video_frame.h"
  26 #include "ppapi/utility/completion_callback_factory.h"
  27
  28 // When compiling natively on Windows, PostMessage can be #define-d to
  29 // something else.
  30 #ifdef PostMessage
  31 #undef PostMessage
  32 #endif
  33
  34 // Use assert as a poor-man's CHECK, even in non-debug mode.
  35 // Since <assert.h> redefines assert on every inclusion (it doesn't use
  36 // include-guards), make sure this is the last file #include'd in this file.
  37 #undef NDEBUG
  38 #include <assert.h>
  39
  40 #define fourcc(a, b, c, d)                                               \
  41   (((uint32_t)(a) << 0) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | \
  42    ((uint32_t)(d) << 24))
  43
  44 namespace {
  45
  46 // IVF container writer. It is possible to parse H264 bitstream using
  47 // NAL units but for VP8 we need a container to at least find encoded
  48 // pictures as well as the picture sizes.
  49 class IVFWriter {
  50  public:
  51   IVFWriter() {}
  52   ~IVFWriter() {}
  53
  54   uint32_t GetFileHeaderSize() const { return 32; }
  55   uint32_t GetFrameHeaderSize() const { return 12; }
  56   uint32_t WriteFileHeader(uint8_t* mem, int32_t width, int32_t height);
  57   uint32_t WriteFrameHeader(uint8_t* mem, uint64_t pts, size_t frame_size);
  58
  59  private:
  60   void PutLE16(uint8_t* mem, int val) const {
  61     mem[0] = (val >> 0) & 0xff;
  62     mem[1] = (val >> 8) & 0xff;
  63   }
  64   void PutLE32(uint8_t* mem, int val) const {
  65     mem[0] = (val >> 0) & 0xff;
  66     mem[1] = (val >> 8) & 0xff;
  67     mem[2] = (val >> 16) & 0xff;
  68     mem[3] = (val >> 24) & 0xff;
  69   }
  70 };
  71
  72 uint32_t IVFWriter::WriteFileHeader(uint8_t* mem,
  73                                     int32_t width,
  74                                     int32_t height) {
  75   mem[0] = 'D';
  76   mem[1] = 'K';
  77   mem[2] = 'I';
  78   mem[3] = 'F';
  79   PutLE16(mem + 4, 0);                               // version
  80   PutLE16(mem + 6, 32);                              // header size
  81   PutLE32(mem + 8, fourcc('V', 'P', '8', '0'));      // fourcc
  82   PutLE16(mem + 12, static_cast<uint16_t>(width));   // width
  83   PutLE16(mem + 14, static_cast<uint16_t>(height));  // height
  84   PutLE32(mem + 16, 30);                             // rate
  85   PutLE32(mem + 20, 1);                              // scale
  86   PutLE32(mem + 24, 0xffffffff);                     // length
  87   PutLE32(mem + 28, 0);                              // unused
  88
  89   return 32;
  90 }
  91
  92 uint32_t IVFWriter::WriteFrameHeader(uint8_t* mem,
  93                                      uint64_t pts,
  94                                      size_t frame_size) {
  95   PutLE32(mem, (int)frame_size);
  96   PutLE32(mem + 4, (int)(pts & 0xFFFFFFFF));
  97   PutLE32(mem + 8, (int)(pts >> 32));
  98
  99   return 12;
 100 }
 101
 102 // This object is the global object representing this plugin library as long
 103 // as it is loaded.
 104 class VideoEncoderModule : public pp::Module {
 105  public:
 106   VideoEncoderModule() : pp::Module() {}
 107   virtual ~VideoEncoderModule() {}
 108
 109   virtual pp::Instance* CreateInstance(PP_Instance instance);
 110 };
 111
 112 class VideoEncoderInstance : public pp::Instance {
 113  public:
 114   VideoEncoderInstance(PP_Instance instance, pp::Module* module);
 115   virtual ~VideoEncoderInstance();
 116
 117   // pp::Instance implementation.
 118   virtual void HandleMessage(const pp::Var& var_message);
 119
 120  private:
 121   void AddVideoProfile(PP_VideoProfile profile, const std::string& profile_str);
 122   void InitializeVideoProfiles();
 123   PP_VideoProfile VideoProfileFromString(const std::string& str);
 124   std::string VideoProfileToString(PP_VideoProfile profile);
 125
 126   void ConfigureTrack();
 127   void OnConfiguredTrack(int32_t result);
 128   void ProbeEncoder();
 129   void OnEncoderProbed(int32_t result,
 130                        const std::vector<PP_VideoProfileDescription> profiles);
 131   void StartEncoder();
 132   void OnInitializedEncoder(int32_t result);
 133   void ScheduleNextEncode();
 134   void GetEncoderFrameTick(int32_t result);
 135   void GetEncoderFrame(const pp::VideoFrame& track_frame);
 136   void OnEncoderFrame(int32_t result,
 137                       pp::VideoFrame encoder_frame,
 138                       pp::VideoFrame track_frame);
 139   int32_t CopyVideoFrame(pp::VideoFrame dest, pp::VideoFrame src);
 140   void EncodeFrame(const pp::VideoFrame& frame);
 141   void OnEncodeDone(int32_t result);
 142   void OnGetBitstreamBuffer(int32_t result, PP_BitstreamBuffer buffer);
 143   void StartTrackFrames();
 144   void StopTrackFrames();
 145   void OnTrackFrame(int32_t result, pp::VideoFrame frame);
 146
 147   void StopEncode();
 148
 149   void LogError(int32_t error, const std::string& message);
 150   void Log(const std::string& message);
 151
 152   void PostDataMessage(const void* buffer, uint32_t size);
 153   void PostSignalMessage(const char* name);
 154
 155   typedef std::map<std::string, PP_VideoProfile> VideoProfileFromStringMap;
 156   VideoProfileFromStringMap profile_from_string_;
 157
 158   typedef std::map<PP_VideoProfile, std::string> VideoProfileToStringMap;
 159   VideoProfileToStringMap profile_to_string_;
 160
 161   bool is_encoding_;
 162   bool is_receiving_track_frames_;
 163
 164   pp::VideoEncoder video_encoder_;
 165   pp::MediaStreamVideoTrack video_track_;
 166   pp::CompletionCallbackFactory<VideoEncoderInstance> callback_factory_;
 167
 168   PP_VideoProfile video_profile_;
 169   PP_VideoFrame_Format frame_format_;
 170
 171   pp::Size requested_size_;
 172   pp::Size frame_size_;
 173   pp::Size encoder_size_;
 174   uint32_t encoded_frames_;
 175
 176   pp::VideoFrame current_track_frame_;
 177
 178   IVFWriter ivf_writer_;
 179 };
 180
 181 VideoEncoderInstance::VideoEncoderInstance(PP_Instance instance,
 182                                            pp::Module* module)
 183     : pp::Instance(instance),
 184       is_encoding_(false),
 185       callback_factory_(this),
 186 #if defined(USE_VP8_INSTEAD_OF_H264)
 187       video_profile_(PP_VIDEOPROFILE_VP8_ANY),
 188 #else
 189       video_profile_(PP_VIDEOPROFILE_H264MAIN),
 190 #endif
 191       frame_format_(PP_VIDEOFRAME_FORMAT_I420),
 192       encoded_frames_(0) {
 193   InitializeVideoProfiles();
 194   ProbeEncoder();
 195 }
 196
 197 VideoEncoderInstance::~VideoEncoderInstance() {
 198 }
 199
 200 void VideoEncoderInstance::AddVideoProfile(PP_VideoProfile profile,
 201                                            const std::string& profile_str) {
 202   profile_to_string_.insert(std::make_pair(profile, profile_str));
 203   profile_from_string_.insert(std::make_pair(profile_str, profile));
 204 }
 205
 206 void VideoEncoderInstance::InitializeVideoProfiles() {
 207   AddVideoProfile(PP_VIDEOPROFILE_H264BASELINE, "h264baseline");
 208   AddVideoProfile(PP_VIDEOPROFILE_H264MAIN, "h264main");
 209   AddVideoProfile(PP_VIDEOPROFILE_H264EXTENDED, "h264extended");
 210   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH, "h264high");
 211   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH10PROFILE, "h264high10");
 212   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH422PROFILE, "h264high422");
 213   AddVideoProfile(PP_VIDEOPROFILE_H264HIGH444PREDICTIVEPROFILE,
 214                   "h264high444predictive");
 215   AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEBASELINE, "h264scalablebaseline");
 216   AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEHIGH, "h264scalablehigh");
 217   AddVideoProfile(PP_VIDEOPROFILE_H264STEREOHIGH, "h264stereohigh");
 218   AddVideoProfile(PP_VIDEOPROFILE_H264MULTIVIEWHIGH, "h264multiviewhigh");
 219   AddVideoProfile(PP_VIDEOPROFILE_VP8_ANY, "vp8");
 220   AddVideoProfile(PP_VIDEOPROFILE_VP9_ANY, "vp9");
 221 }
 222
 223 PP_VideoProfile VideoEncoderInstance::VideoProfileFromString(
 224     const std::string& str) {
 225   VideoProfileFromStringMap::iterator it = profile_from_string_.find(str);
 226   if (it == profile_from_string_.end())
 227     return PP_VIDEOPROFILE_VP8_ANY;
 228   return it->second;
 229 }
 230
 231 std::string VideoEncoderInstance::VideoProfileToString(
 232     PP_VideoProfile profile) {
 233   VideoProfileToStringMap::iterator it = profile_to_string_.find(profile);
 234   if (it == profile_to_string_.end())
 235     return "unknown";
 236   return it->second;
 237 }
 238
 239 void VideoEncoderInstance::ConfigureTrack() {
 240   if (encoder_size_.IsEmpty())
 241     frame_size_ = requested_size_;
 242   else
 243     frame_size_ = encoder_size_;
 244
 245   int32_t attrib_list[] = {PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT,
 246                            frame_format_,
 247                            PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH,
 248                            frame_size_.width(),
 249                            PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT,
 250                            frame_size_.height(),
 251                            PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE};
 252
 253   pp::VarDictionary dict;
 254   dict.Set(pp::Var("status"), pp::Var("configuring video track"));
 255   dict.Set(pp::Var("width"), pp::Var(frame_size_.width()));
 256   dict.Set(pp::Var("height"), pp::Var(frame_size_.height()));
 257   PostMessage(dict);
 258
 259   video_track_.Configure(
 260       attrib_list,
 261       callback_factory_.NewCallback(&VideoEncoderInstance::OnConfiguredTrack));
 262 }
 263
 264 void VideoEncoderInstance::OnConfiguredTrack(int32_t result) {
 265   if (result != PP_OK) {
 266     LogError(result, "Cannot configure track");
 267     return;
 268   }
 269
 270   if (is_encoding_) {
 271     StartTrackFrames();
 272     ScheduleNextEncode();
 273   } else
 274     StartEncoder();
 275 }
 276
 277 void VideoEncoderInstance::ProbeEncoder() {
 278   video_encoder_ = pp::VideoEncoder(this);
 279   video_encoder_.GetSupportedProfiles(callback_factory_.NewCallbackWithOutput(
 280       &VideoEncoderInstance::OnEncoderProbed));
 281 }
 282
 283 void VideoEncoderInstance::OnEncoderProbed(
 284     int32_t result,
 285     const std::vector<PP_VideoProfileDescription> profiles) {
 286   pp::VarDictionary dict;
 287   dict.Set(pp::Var("name"), pp::Var("supportedProfiles"));
 288   pp::VarArray js_profiles;
 289   dict.Set(pp::Var("profiles"), js_profiles);
 290
 291   if (result < 0) {
 292     LogError(result, "Cannot get supported profiles");
 293     PostMessage(dict);
 294   }
 295
 296   int32_t idx = 0;
 297   for (const PP_VideoProfileDescription& profile : profiles)
 298     js_profiles.Set(idx++, pp::Var(VideoProfileToString(profile.profile)));
 299   PostMessage(dict);
 300 }
 301
 302 void VideoEncoderInstance::StartEncoder() {
 303   video_encoder_ = pp::VideoEncoder(this);
 304
 305   int32_t error = video_encoder_.Initialize(
 306       frame_format_, frame_size_, video_profile_, 2000000,
 307       PP_HARDWAREACCELERATION_WITHFALLBACK,
 308       callback_factory_.NewCallback(
 309           &VideoEncoderInstance::OnInitializedEncoder));
 310   if (error != PP_OK_COMPLETIONPENDING) {
 311     LogError(error, "Cannot initialize encoder");
 312     return;
 313   }
 314 }
 315
 316 void VideoEncoderInstance::OnInitializedEncoder(int32_t result) {
 317   if (result != PP_OK) {
 318     LogError(result, "Encoder initialization failed");
 319     return;
 320   }
 321
 322   is_encoding_ = true;
 323   PostSignalMessage("started");
 324
 325   if (video_encoder_.GetFrameCodedSize(&encoder_size_) != PP_OK) {
 326     LogError(result, "Cannot get encoder coded frame size");
 327     return;
 328   }
 329
 330   video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
 331       &VideoEncoderInstance::OnGetBitstreamBuffer));
 332
 333   if (encoder_size_ != frame_size_)
 334     ConfigureTrack();
 335   else {
 336     StartTrackFrames();
 337     ScheduleNextEncode();
 338   }
 339 }
 340
 341 void VideoEncoderInstance::ScheduleNextEncode() {
 342   pp::Module::Get()->core()->CallOnMainThread(
 343       1000 / 30,
 344       callback_factory_.NewCallback(&VideoEncoderInstance::GetEncoderFrameTick),
 345       0);
 346 }
 347
 348 void VideoEncoderInstance::GetEncoderFrameTick(int32_t result) {
 349   if (is_encoding_) {
 350     if (!current_track_frame_.is_null()) {
 351       pp::VideoFrame frame = current_track_frame_;
 352       current_track_frame_.detach();
 353       GetEncoderFrame(frame);
 354     }
 355     ScheduleNextEncode();
 356   }
 357 }
 358
 359 void VideoEncoderInstance::GetEncoderFrame(const pp::VideoFrame& track_frame) {
 360   video_encoder_.GetVideoFrame(callback_factory_.NewCallbackWithOutput(
 361       &VideoEncoderInstance::OnEncoderFrame, track_frame));
 362 }
 363
 364 void VideoEncoderInstance::OnEncoderFrame(int32_t result,
 365                                           pp::VideoFrame encoder_frame,
 366                                           pp::VideoFrame track_frame) {
 367   if (result == PP_ERROR_ABORTED) {
 368     video_track_.RecycleFrame(track_frame);
 369     return;
 370   }
 371   if (result != PP_OK) {
 372     video_track_.RecycleFrame(track_frame);
 373     LogError(result, "Cannot get video frame from video encoder");
 374     return;
 375   }
 376
 377   track_frame.GetSize(&frame_size_);
 378
 379   if (frame_size_ != encoder_size_) {
 380     video_track_.RecycleFrame(track_frame);
 381     LogError(PP_ERROR_FAILED, "MediaStreamVideoTrack frame size incorrect");
 382     return;
 383   }
 384
 385   if (CopyVideoFrame(encoder_frame, track_frame) == PP_OK)
 386     EncodeFrame(encoder_frame);
 387   video_track_.RecycleFrame(track_frame);
 388 }
 389
 390 int32_t VideoEncoderInstance::CopyVideoFrame(pp::VideoFrame dest,
 391                                              pp::VideoFrame src) {
 392   if (dest.GetDataBufferSize() < src.GetDataBufferSize()) {
 393     std::ostringstream oss;
 394     oss << "Incorrect destination video frame buffer size : "
 395         << dest.GetDataBufferSize() << " < " << src.GetDataBufferSize();
 396     LogError(PP_ERROR_FAILED, oss.str());
 397     return PP_ERROR_FAILED;
 398   }
 399
 400   dest.SetTimestamp(src.GetTimestamp());
 401   memcpy(dest.GetDataBuffer(), src.GetDataBuffer(), src.GetDataBufferSize());
 402   return PP_OK;
 403 }
 404
 405 void VideoEncoderInstance::EncodeFrame(const pp::VideoFrame& frame) {
 406   video_encoder_.Encode(
 407       frame, PP_FALSE,
 408       callback_factory_.NewCallback(&VideoEncoderInstance::OnEncodeDone));
 409 }
 410
 411 void VideoEncoderInstance::OnEncodeDone(int32_t result) {
 412   if (result == PP_ERROR_ABORTED)
 413     return;
 414   if (result != PP_OK)
 415     LogError(result, "Encode failed");
 416 }
 417
 418 void VideoEncoderInstance::OnGetBitstreamBuffer(int32_t result,
 419                                                 PP_BitstreamBuffer buffer) {
 420   if (result == PP_ERROR_ABORTED)
 421     return;
 422   if (result != PP_OK) {
 423     LogError(result, "Cannot get bitstream buffer");
 424     return;
 425   }
 426
 427   encoded_frames_++;
 428   PostDataMessage(buffer.buffer, buffer.size);
 429   video_encoder_.RecycleBitstreamBuffer(buffer);
 430
 431   video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
 432       &VideoEncoderInstance::OnGetBitstreamBuffer));
 433 }
 434
 435 void VideoEncoderInstance::StartTrackFrames() {
 436   is_receiving_track_frames_ = true;
 437   video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 438       &VideoEncoderInstance::OnTrackFrame));
 439 }
 440
 441 void VideoEncoderInstance::StopTrackFrames() {
 442   is_receiving_track_frames_ = false;
 443   if (!current_track_frame_.is_null()) {
 444     video_track_.RecycleFrame(current_track_frame_);
 445     current_track_frame_.detach();
 446   }
 447 }
 448
 449 void VideoEncoderInstance::OnTrackFrame(int32_t result, pp::VideoFrame frame) {
 450   if (result == PP_ERROR_ABORTED)
 451     return;
 452
 453   if (!current_track_frame_.is_null()) {
 454     video_track_.RecycleFrame(current_track_frame_);
 455     current_track_frame_.detach();
 456   }
 457
 458   if (result != PP_OK) {
 459     LogError(result, "Cannot get video frame from video track");
 460     return;
 461   }
 462
 463   current_track_frame_ = frame;
 464   if (is_receiving_track_frames_)
 465     video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
 466         &VideoEncoderInstance::OnTrackFrame));
 467 }
 468
 469 void VideoEncoderInstance::StopEncode() {
 470   video_encoder_.Close();
 471   StopTrackFrames();
 472   video_track_.Close();
 473   is_encoding_ = false;
 474   encoded_frames_ = 0;
 475 }
 476
 477 //
 478
 479 void VideoEncoderInstance::HandleMessage(const pp::Var& var_message) {
 480   if (!var_message.is_dictionary()) {
 481     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
 482     return;
 483   }
 484
 485   pp::VarDictionary dict_message(var_message);
 486   std::string command = dict_message.Get("command").AsString();
 487
 488   if (command == "start") {
 489     requested_size_ = pp::Size(dict_message.Get("width").AsInt(),
 490                                dict_message.Get("height").AsInt());
 491     pp::Var var_track = dict_message.Get("track");
 492     if (!var_track.is_resource()) {
 493       LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Given track is not a resource"));
 494       return;
 495     }
 496     pp::Resource resource_track = var_track.AsResource();
 497     video_track_ = pp::MediaStreamVideoTrack(resource_track);
 498     video_encoder_ = pp::VideoEncoder();
 499     video_profile_ = VideoProfileFromString(
 500         dict_message.Get("profile").AsString());
 501     ConfigureTrack();
 502   } else if (command == "stop") {
 503     StopEncode();
 504     PostSignalMessage("stopped");
 505   } else {
 506     LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
 507   }
 508 }
 509
 510 void VideoEncoderInstance::PostDataMessage(const void* buffer, uint32_t size) {
 511   pp::VarDictionary dictionary;
 512
 513   dictionary.Set(pp::Var("name"), pp::Var("data"));
 514
 515   pp::VarArrayBuffer array_buffer;
 516   uint8_t* data_ptr;
 517   uint32_t data_offset = 0;
 518   if (video_profile_ == PP_VIDEOPROFILE_VP8_ANY ||
 519       video_profile_ == PP_VIDEOPROFILE_VP9_ANY) {
 520     uint32_t frame_offset = 0;
 521     if (encoded_frames_ == 1) {
 522       array_buffer = pp::VarArrayBuffer(
 523           size + ivf_writer_.GetFileHeaderSize() +
 524           ivf_writer_.GetFrameHeaderSize());
 525       data_ptr = static_cast<uint8_t*>(array_buffer.Map());
 526       frame_offset = ivf_writer_.WriteFileHeader(
 527           data_ptr, frame_size_.width(), frame_size_.height());
 528     } else {
 529       array_buffer = pp::VarArrayBuffer(
 530           size + ivf_writer_.GetFrameHeaderSize());
 531       data_ptr = static_cast<uint8_t*>(array_buffer.Map());
 532     }
 533     data_offset = frame_offset +
 534       ivf_writer_.WriteFrameHeader(data_ptr + frame_offset,
 535                                    encoded_frames_,
 536                                    size);
 537   } else {
 538     array_buffer = pp::VarArrayBuffer(size);
 539     data_ptr = static_cast<uint8_t*>(array_buffer.Map());
 540   }
 541
 542   memcpy(data_ptr + data_offset, buffer, size);
 543   array_buffer.Unmap();
 544   dictionary.Set(pp::Var("data"), array_buffer);
 545
 546   PostMessage(dictionary);
 547 }
 548
 549 void VideoEncoderInstance::PostSignalMessage(const char* name) {
 550   pp::VarDictionary dictionary;
 551   dictionary.Set(pp::Var("name"), pp::Var(name));
 552
 553   PostMessage(dictionary);
 554 }
 555
 556 void VideoEncoderInstance::LogError(int32_t error, const std::string& message) {
 557   std::string msg("Error: ");
 558   msg.append(pp::Var(error).DebugString());
 559   msg.append(" : ");
 560   msg.append(message);
 561   LogToConsole(PP_LOGLEVEL_ERROR, pp::Var(msg));
 562 }
 563
 564 void VideoEncoderInstance::Log(const std::string& message) {
 565   LogToConsole(PP_LOGLEVEL_LOG, pp::Var(message));
 566 }
 567
 568 pp::Instance* VideoEncoderModule::CreateInstance(PP_Instance instance) {
 569   return new VideoEncoderInstance(instance, this);
 570 }
 571
 572 }  // anonymous namespace
 573
 574 namespace pp {
 575 // Factory function for your specialization of the Module object.
 576 Module* CreateModule() {
 577   return new VideoEncoderModule();
 578 }
 579 }  // namespace pp