Merge Chromium + Blink git repositories
[chromium-blink-merge.git] / native_client_sdk / src / examples / api / video_encode / video_encode.cc
blob00674bfd33b7b3b295b6f6d8183412b86dedad78
1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include <math.h>
6 #include <stdio.h>
7 #include <string.h>
9 #include <algorithm>
10 #include <deque>
11 #include <iostream>
12 #include <map>
13 #include <sstream>
14 #include <vector>
16 #include "ppapi/c/pp_errors.h"
17 #include "ppapi/c/ppb_console.h"
18 #include "ppapi/cpp/input_event.h"
19 #include "ppapi/cpp/instance.h"
20 #include "ppapi/cpp/media_stream_video_track.h"
21 #include "ppapi/cpp/module.h"
22 #include "ppapi/cpp/rect.h"
23 #include "ppapi/cpp/var.h"
24 #include "ppapi/cpp/var_array_buffer.h"
25 #include "ppapi/cpp/var_dictionary.h"
26 #include "ppapi/cpp/video_encoder.h"
27 #include "ppapi/cpp/video_frame.h"
28 #include "ppapi/utility/completion_callback_factory.h"
30 // When compiling natively on Windows, PostMessage, min and max can be
31 // #define-d to something else.
32 #ifdef WIN32
33 #undef min
34 #undef max
35 #undef PostMessage
36 #endif
38 // Use assert as a poor-man's CHECK, even in non-debug mode.
39 // Since <assert.h> redefines assert on every inclusion (it doesn't use
40 // include-guards), make sure this is the last file #include'd in this file.
41 #undef NDEBUG
42 #include <assert.h>
44 #define fourcc(a, b, c, d) \
45 (((uint32_t)(a) << 0) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | \
46 ((uint32_t)(d) << 24))
48 namespace {
50 double clamp(double min, double max, double value) {
51 return std::max(std::min(value, max), min);
54 std::string ToUpperString(const std::string& str) {
55 std::string ret;
56 for (uint32_t i = 0; i < str.size(); i++)
57 ret.push_back(static_cast<char>(toupper(str[i])));
58 return ret;
61 // IVF container writer. It is possible to parse H264 bitstream using
62 // NAL units but for VP8 we need a container to at least find encoded
63 // pictures as well as the picture sizes.
64 class IVFWriter {
65 public:
66 IVFWriter() {}
67 ~IVFWriter() {}
69 uint32_t GetFileHeaderSize() const { return 32; }
70 uint32_t GetFrameHeaderSize() const { return 12; }
71 uint32_t WriteFileHeader(uint8_t* mem,
72 const std::string& codec,
73 int32_t width,
74 int32_t height);
75 uint32_t WriteFrameHeader(uint8_t* mem, uint64_t pts, size_t frame_size);
77 private:
78 void PutLE16(uint8_t* mem, int val) const {
79 mem[0] = (val >> 0) & 0xff;
80 mem[1] = (val >> 8) & 0xff;
82 void PutLE32(uint8_t* mem, int val) const {
83 mem[0] = (val >> 0) & 0xff;
84 mem[1] = (val >> 8) & 0xff;
85 mem[2] = (val >> 16) & 0xff;
86 mem[3] = (val >> 24) & 0xff;
90 uint32_t IVFWriter::WriteFileHeader(uint8_t* mem,
91 const std::string& codec,
92 int32_t width,
93 int32_t height) {
94 mem[0] = 'D';
95 mem[1] = 'K';
96 mem[2] = 'I';
97 mem[3] = 'F';
98 PutLE16(mem + 4, 0); // version
99 PutLE16(mem + 6, 32); // header size
100 PutLE32(mem + 8, fourcc(codec[0], codec[1], codec[2], '0')); // fourcc
101 PutLE16(mem + 12, static_cast<uint16_t>(width)); // width
102 PutLE16(mem + 14, static_cast<uint16_t>(height)); // height
103 PutLE32(mem + 16, 1000); // rate
104 PutLE32(mem + 20, 1); // scale
105 PutLE32(mem + 24, 0xffffffff); // length
106 PutLE32(mem + 28, 0); // unused
108 return 32;
111 uint32_t IVFWriter::WriteFrameHeader(uint8_t* mem,
112 uint64_t pts,
113 size_t frame_size) {
114 PutLE32(mem, (int)frame_size);
115 PutLE32(mem + 4, (int)(pts & 0xFFFFFFFF));
116 PutLE32(mem + 8, (int)(pts >> 32));
118 return 12;
121 // This object is the global object representing this plugin library as long
122 // as it is loaded.
123 class VideoEncoderModule : public pp::Module {
124 public:
125 VideoEncoderModule() : pp::Module() {}
126 virtual ~VideoEncoderModule() {}
128 virtual pp::Instance* CreateInstance(PP_Instance instance);
131 class VideoEncoderInstance : public pp::Instance {
132 public:
133 VideoEncoderInstance(PP_Instance instance, pp::Module* module);
134 virtual ~VideoEncoderInstance();
136 // pp::Instance implementation.
137 virtual void HandleMessage(const pp::Var& var_message);
139 private:
140 void AddVideoProfile(PP_VideoProfile profile, const std::string& profile_str);
141 void InitializeVideoProfiles();
142 PP_VideoProfile VideoProfileFromString(const std::string& str);
143 std::string VideoProfileToString(PP_VideoProfile profile);
145 void ConfigureTrack();
146 void OnConfiguredTrack(int32_t result);
147 void ProbeEncoder();
148 void OnEncoderProbed(int32_t result,
149 const std::vector<PP_VideoProfileDescription> profiles);
150 void StartEncoder();
151 void OnInitializedEncoder(int32_t result);
152 void ScheduleNextEncode();
153 void GetEncoderFrameTick(int32_t result);
154 void GetEncoderFrame(const pp::VideoFrame& track_frame);
155 void OnEncoderFrame(int32_t result,
156 pp::VideoFrame encoder_frame,
157 pp::VideoFrame track_frame);
158 int32_t CopyVideoFrame(pp::VideoFrame dest, pp::VideoFrame src);
159 void EncodeFrame(const pp::VideoFrame& frame);
160 void OnEncodeDone(int32_t result);
161 void OnGetBitstreamBuffer(int32_t result, PP_BitstreamBuffer buffer);
162 void StartTrackFrames();
163 void StopTrackFrames();
164 void OnTrackFrame(int32_t result, pp::VideoFrame frame);
166 void StopEncode();
168 void LogError(int32_t error, const std::string& message);
169 void Log(const std::string& message);
171 void PostDataMessage(const void* buffer, uint32_t size);
173 typedef std::map<std::string, PP_VideoProfile> VideoProfileFromStringMap;
174 VideoProfileFromStringMap profile_from_string_;
176 typedef std::map<PP_VideoProfile, std::string> VideoProfileToStringMap;
177 VideoProfileToStringMap profile_to_string_;
179 bool is_encoding_;
180 bool is_encode_ticking_;
181 bool is_receiving_track_frames_;
183 pp::VideoEncoder video_encoder_;
184 pp::MediaStreamVideoTrack video_track_;
185 pp::CompletionCallbackFactory<VideoEncoderInstance> callback_factory_;
187 PP_VideoProfile video_profile_;
188 PP_VideoFrame_Format frame_format_;
190 pp::Size requested_size_;
191 pp::Size frame_size_;
192 pp::Size encoder_size_;
193 uint32_t encoded_frames_;
195 std::deque<uint64_t> frames_timestamps_;
197 pp::VideoFrame current_track_frame_;
199 IVFWriter ivf_writer_;
201 PP_Time last_encode_tick_;
204 VideoEncoderInstance::VideoEncoderInstance(PP_Instance instance,
205 pp::Module* module)
206 : pp::Instance(instance),
207 is_encoding_(false),
208 is_encode_ticking_(false),
209 callback_factory_(this),
210 #if defined(USE_VP8_INSTEAD_OF_H264)
211 video_profile_(PP_VIDEOPROFILE_VP8_ANY),
212 #else
213 video_profile_(PP_VIDEOPROFILE_H264MAIN),
214 #endif
215 frame_format_(PP_VIDEOFRAME_FORMAT_I420),
216 encoded_frames_(0),
217 last_encode_tick_(0) {
218 InitializeVideoProfiles();
219 ProbeEncoder();
222 VideoEncoderInstance::~VideoEncoderInstance() {
225 void VideoEncoderInstance::AddVideoProfile(PP_VideoProfile profile,
226 const std::string& profile_str) {
227 profile_to_string_.insert(std::make_pair(profile, profile_str));
228 profile_from_string_.insert(std::make_pair(profile_str, profile));
231 void VideoEncoderInstance::InitializeVideoProfiles() {
232 AddVideoProfile(PP_VIDEOPROFILE_H264BASELINE, "h264baseline");
233 AddVideoProfile(PP_VIDEOPROFILE_H264MAIN, "h264main");
234 AddVideoProfile(PP_VIDEOPROFILE_H264EXTENDED, "h264extended");
235 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH, "h264high");
236 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH10PROFILE, "h264high10");
237 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH422PROFILE, "h264high422");
238 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH444PREDICTIVEPROFILE,
239 "h264high444predictive");
240 AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEBASELINE, "h264scalablebaseline");
241 AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEHIGH, "h264scalablehigh");
242 AddVideoProfile(PP_VIDEOPROFILE_H264STEREOHIGH, "h264stereohigh");
243 AddVideoProfile(PP_VIDEOPROFILE_H264MULTIVIEWHIGH, "h264multiviewhigh");
244 AddVideoProfile(PP_VIDEOPROFILE_VP8_ANY, "vp8");
245 AddVideoProfile(PP_VIDEOPROFILE_VP9_ANY, "vp9");
248 PP_VideoProfile VideoEncoderInstance::VideoProfileFromString(
249 const std::string& str) {
250 VideoProfileFromStringMap::iterator it = profile_from_string_.find(str);
251 if (it == profile_from_string_.end())
252 return PP_VIDEOPROFILE_VP8_ANY;
253 return it->second;
256 std::string VideoEncoderInstance::VideoProfileToString(
257 PP_VideoProfile profile) {
258 VideoProfileToStringMap::iterator it = profile_to_string_.find(profile);
259 if (it == profile_to_string_.end())
260 return "unknown";
261 return it->second;
264 void VideoEncoderInstance::ConfigureTrack() {
265 if (encoder_size_.IsEmpty())
266 frame_size_ = requested_size_;
267 else
268 frame_size_ = encoder_size_;
270 int32_t attrib_list[] = {PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT,
271 frame_format_,
272 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH,
273 frame_size_.width(),
274 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT,
275 frame_size_.height(),
276 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE};
278 video_track_.Configure(
279 attrib_list,
280 callback_factory_.NewCallback(&VideoEncoderInstance::OnConfiguredTrack));
283 void VideoEncoderInstance::OnConfiguredTrack(int32_t result) {
284 if (result != PP_OK) {
285 LogError(result, "Cannot configure track");
286 return;
289 if (is_encoding_) {
290 StartTrackFrames();
291 ScheduleNextEncode();
292 } else
293 StartEncoder();
296 void VideoEncoderInstance::ProbeEncoder() {
297 video_encoder_ = pp::VideoEncoder(this);
298 video_encoder_.GetSupportedProfiles(callback_factory_.NewCallbackWithOutput(
299 &VideoEncoderInstance::OnEncoderProbed));
302 void VideoEncoderInstance::OnEncoderProbed(
303 int32_t result,
304 const std::vector<PP_VideoProfileDescription> profiles) {
305 pp::VarDictionary dict;
306 dict.Set(pp::Var("name"), pp::Var("supportedProfiles"));
307 pp::VarArray js_profiles;
308 dict.Set(pp::Var("profiles"), js_profiles);
310 if (result < 0) {
311 LogError(result, "Cannot get supported profiles");
312 PostMessage(dict);
315 int32_t idx = 0;
316 for (uint32_t i = 0; i < profiles.size(); i++) {
317 const PP_VideoProfileDescription& profile = profiles[i];
318 js_profiles.Set(idx++, pp::Var(VideoProfileToString(profile.profile)));
320 PostMessage(dict);
323 void VideoEncoderInstance::StartEncoder() {
324 video_encoder_ = pp::VideoEncoder(this);
325 frames_timestamps_.clear();
327 int32_t error = video_encoder_.Initialize(
328 frame_format_, frame_size_, video_profile_, 2000000,
329 PP_HARDWAREACCELERATION_WITHFALLBACK,
330 callback_factory_.NewCallback(
331 &VideoEncoderInstance::OnInitializedEncoder));
332 if (error != PP_OK_COMPLETIONPENDING) {
333 LogError(error, "Cannot initialize encoder");
334 return;
338 void VideoEncoderInstance::OnInitializedEncoder(int32_t result) {
339 if (result != PP_OK) {
340 LogError(result, "Encoder initialization failed");
341 return;
344 is_encoding_ = true;
345 Log("started");
347 if (video_encoder_.GetFrameCodedSize(&encoder_size_) != PP_OK) {
348 LogError(result, "Cannot get encoder coded frame size");
349 return;
352 video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
353 &VideoEncoderInstance::OnGetBitstreamBuffer));
355 if (encoder_size_ != frame_size_)
356 ConfigureTrack();
357 else {
358 StartTrackFrames();
359 ScheduleNextEncode();
363 void VideoEncoderInstance::ScheduleNextEncode() {
364 // Avoid scheduling more than once at a time.
365 if (is_encode_ticking_)
366 return;
368 PP_Time now = pp::Module::Get()->core()->GetTime();
369 PP_Time tick = 1.0 / 30;
370 // If the callback was triggered late, we need to account for that
371 // delay for the next tick.
372 PP_Time delta = tick - clamp(0, tick, now - last_encode_tick_ - tick);
374 pp::Module::Get()->core()->CallOnMainThread(
375 delta * 1000,
376 callback_factory_.NewCallback(&VideoEncoderInstance::GetEncoderFrameTick),
379 last_encode_tick_ = now;
380 is_encode_ticking_ = true;
383 void VideoEncoderInstance::GetEncoderFrameTick(int32_t result) {
384 is_encode_ticking_ = false;
386 if (is_encoding_) {
387 if (!current_track_frame_.is_null()) {
388 pp::VideoFrame frame = current_track_frame_;
389 current_track_frame_.detach();
390 GetEncoderFrame(frame);
392 ScheduleNextEncode();
396 void VideoEncoderInstance::GetEncoderFrame(const pp::VideoFrame& track_frame) {
397 video_encoder_.GetVideoFrame(callback_factory_.NewCallbackWithOutput(
398 &VideoEncoderInstance::OnEncoderFrame, track_frame));
401 void VideoEncoderInstance::OnEncoderFrame(int32_t result,
402 pp::VideoFrame encoder_frame,
403 pp::VideoFrame track_frame) {
404 if (result == PP_ERROR_ABORTED) {
405 video_track_.RecycleFrame(track_frame);
406 return;
408 if (result != PP_OK) {
409 video_track_.RecycleFrame(track_frame);
410 LogError(result, "Cannot get video frame from video encoder");
411 return;
414 track_frame.GetSize(&frame_size_);
416 if (frame_size_ != encoder_size_) {
417 video_track_.RecycleFrame(track_frame);
418 LogError(PP_ERROR_FAILED, "MediaStreamVideoTrack frame size incorrect");
419 return;
422 if (CopyVideoFrame(encoder_frame, track_frame) == PP_OK)
423 EncodeFrame(encoder_frame);
424 video_track_.RecycleFrame(track_frame);
427 int32_t VideoEncoderInstance::CopyVideoFrame(pp::VideoFrame dest,
428 pp::VideoFrame src) {
429 if (dest.GetDataBufferSize() < src.GetDataBufferSize()) {
430 std::ostringstream oss;
431 oss << "Incorrect destination video frame buffer size : "
432 << dest.GetDataBufferSize() << " < " << src.GetDataBufferSize();
433 LogError(PP_ERROR_FAILED, oss.str());
434 return PP_ERROR_FAILED;
437 dest.SetTimestamp(src.GetTimestamp());
438 memcpy(dest.GetDataBuffer(), src.GetDataBuffer(), src.GetDataBufferSize());
439 return PP_OK;
442 void VideoEncoderInstance::EncodeFrame(const pp::VideoFrame& frame) {
443 frames_timestamps_.push_back(
444 static_cast<uint64_t>(frame.GetTimestamp() * 1000));
445 video_encoder_.Encode(
446 frame, PP_FALSE,
447 callback_factory_.NewCallback(&VideoEncoderInstance::OnEncodeDone));
450 void VideoEncoderInstance::OnEncodeDone(int32_t result) {
451 if (result == PP_ERROR_ABORTED)
452 return;
453 if (result != PP_OK)
454 LogError(result, "Encode failed");
457 void VideoEncoderInstance::OnGetBitstreamBuffer(int32_t result,
458 PP_BitstreamBuffer buffer) {
459 if (result == PP_ERROR_ABORTED)
460 return;
461 if (result != PP_OK) {
462 LogError(result, "Cannot get bitstream buffer");
463 return;
466 encoded_frames_++;
467 PostDataMessage(buffer.buffer, buffer.size);
468 video_encoder_.RecycleBitstreamBuffer(buffer);
470 video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
471 &VideoEncoderInstance::OnGetBitstreamBuffer));
474 void VideoEncoderInstance::StartTrackFrames() {
475 is_receiving_track_frames_ = true;
476 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
477 &VideoEncoderInstance::OnTrackFrame));
480 void VideoEncoderInstance::StopTrackFrames() {
481 is_receiving_track_frames_ = false;
482 if (!current_track_frame_.is_null()) {
483 video_track_.RecycleFrame(current_track_frame_);
484 current_track_frame_.detach();
488 void VideoEncoderInstance::OnTrackFrame(int32_t result, pp::VideoFrame frame) {
489 if (result == PP_ERROR_ABORTED)
490 return;
492 if (!current_track_frame_.is_null()) {
493 video_track_.RecycleFrame(current_track_frame_);
494 current_track_frame_.detach();
497 if (result != PP_OK) {
498 LogError(result, "Cannot get video frame from video track");
499 return;
502 current_track_frame_ = frame;
503 if (is_receiving_track_frames_)
504 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
505 &VideoEncoderInstance::OnTrackFrame));
508 void VideoEncoderInstance::StopEncode() {
509 video_encoder_.Close();
510 StopTrackFrames();
511 video_track_.Close();
512 is_encoding_ = false;
513 encoded_frames_ = 0;
518 void VideoEncoderInstance::HandleMessage(const pp::Var& var_message) {
519 if (!var_message.is_dictionary()) {
520 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
521 return;
524 pp::VarDictionary dict_message(var_message);
525 std::string command = dict_message.Get("command").AsString();
527 if (command == "start") {
528 requested_size_ = pp::Size(dict_message.Get("width").AsInt(),
529 dict_message.Get("height").AsInt());
530 pp::Var var_track = dict_message.Get("track");
531 if (!var_track.is_resource()) {
532 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Given track is not a resource"));
533 return;
535 pp::Resource resource_track = var_track.AsResource();
536 video_track_ = pp::MediaStreamVideoTrack(resource_track);
537 video_encoder_ = pp::VideoEncoder();
538 video_profile_ = VideoProfileFromString(
539 dict_message.Get("profile").AsString());
540 ConfigureTrack();
541 } else if (command == "stop") {
542 StopEncode();
543 Log("stopped");
544 } else {
545 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
549 void VideoEncoderInstance::PostDataMessage(const void* buffer, uint32_t size) {
550 pp::VarDictionary dictionary;
552 dictionary.Set(pp::Var("name"), pp::Var("data"));
554 pp::VarArrayBuffer array_buffer;
555 uint8_t* data_ptr;
556 uint32_t data_offset = 0;
557 if (video_profile_ == PP_VIDEOPROFILE_VP8_ANY ||
558 video_profile_ == PP_VIDEOPROFILE_VP9_ANY) {
559 uint32_t frame_offset = 0;
560 if (encoded_frames_ == 1) {
561 array_buffer = pp::VarArrayBuffer(
562 size + ivf_writer_.GetFileHeaderSize() +
563 ivf_writer_.GetFrameHeaderSize());
564 data_ptr = static_cast<uint8_t*>(array_buffer.Map());
565 frame_offset = ivf_writer_.WriteFileHeader(
566 data_ptr, ToUpperString(VideoProfileToString(video_profile_)),
567 frame_size_.width(), frame_size_.height());
568 } else {
569 array_buffer = pp::VarArrayBuffer(
570 size + ivf_writer_.GetFrameHeaderSize());
571 data_ptr = static_cast<uint8_t*>(array_buffer.Map());
573 uint64_t timestamp = frames_timestamps_.front();
574 frames_timestamps_.pop_front();
575 data_offset =
576 frame_offset +
577 ivf_writer_.WriteFrameHeader(data_ptr + frame_offset, timestamp, size);
578 } else {
579 array_buffer = pp::VarArrayBuffer(size);
580 data_ptr = static_cast<uint8_t*>(array_buffer.Map());
583 memcpy(data_ptr + data_offset, buffer, size);
584 array_buffer.Unmap();
585 dictionary.Set(pp::Var("data"), array_buffer);
587 PostMessage(dictionary);
590 void VideoEncoderInstance::LogError(int32_t error, const std::string& message) {
591 std::string msg("Error: ");
592 msg.append(pp::Var(error).DebugString());
593 msg.append(" : ");
594 msg.append(message);
596 Log(msg);
599 void VideoEncoderInstance::Log(const std::string& message) {
600 pp::VarDictionary dictionary;
601 dictionary.Set(pp::Var("name"), pp::Var("log"));
602 dictionary.Set(pp::Var("message"), pp::Var(message));
604 PostMessage(dictionary);
607 pp::Instance* VideoEncoderModule::CreateInstance(PP_Instance instance) {
608 return new VideoEncoderInstance(instance, this);
611 } // anonymous namespace
613 namespace pp {
614 // Factory function for your specialization of the Module object.
615 Module* CreateModule() {
616 return new VideoEncoderModule();
618 } // namespace pp