Fix crash in SpeechRecognizerImpl introduced in AudioParams refactor.
[chromium-blink-merge.git] / ppapi / examples / video_encode / video_encode.cc
blobbbe67f3dc48442ca88e77976e1d75195f513eff2
1 // Copyright 2015 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include <math.h>
6 #include <stdio.h>
7 #include <string.h>
9 #include <algorithm>
10 #include <deque>
11 #include <iostream>
12 #include <map>
13 #include <sstream>
14 #include <vector>
16 #include "ppapi/c/pp_errors.h"
17 #include "ppapi/c/ppb_console.h"
18 #include "ppapi/cpp/input_event.h"
19 #include "ppapi/cpp/instance.h"
20 #include "ppapi/cpp/media_stream_video_track.h"
21 #include "ppapi/cpp/module.h"
22 #include "ppapi/cpp/rect.h"
23 #include "ppapi/cpp/var.h"
24 #include "ppapi/cpp/var_array_buffer.h"
25 #include "ppapi/cpp/var_dictionary.h"
26 #include "ppapi/cpp/video_encoder.h"
27 #include "ppapi/cpp/video_frame.h"
28 #include "ppapi/utility/completion_callback_factory.h"
30 // When compiling natively on Windows, PostMessage can be #define-d to
31 // something else.
32 #ifdef PostMessage
33 #undef PostMessage
34 #endif
36 // Use assert as a poor-man's CHECK, even in non-debug mode.
37 // Since <assert.h> redefines assert on every inclusion (it doesn't use
38 // include-guards), make sure this is the last file #include'd in this file.
39 #undef NDEBUG
40 #include <assert.h>
42 #define fourcc(a, b, c, d) \
43 (((uint32_t)(a) << 0) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | \
44 ((uint32_t)(d) << 24))
46 namespace {
48 double clamp(double min, double max, double value) {
49 return std::max(std::min(value, max), min);
52 std::string ToUpperString(const std::string& str) {
53 std::string ret;
54 for (uint32_t i = 0; i < str.size(); i++)
55 ret.push_back(static_cast<char>(toupper(str[i])));
56 return ret;
59 // IVF container writer. It is possible to parse H264 bitstream using
60 // NAL units but for VP8 we need a container to at least find encoded
61 // pictures as well as the picture sizes.
62 class IVFWriter {
63 public:
64 IVFWriter() {}
65 ~IVFWriter() {}
67 uint32_t GetFileHeaderSize() const { return 32; }
68 uint32_t GetFrameHeaderSize() const { return 12; }
69 uint32_t WriteFileHeader(uint8_t* mem,
70 const std::string& codec,
71 int32_t width,
72 int32_t height);
73 uint32_t WriteFrameHeader(uint8_t* mem, uint64_t pts, size_t frame_size);
75 private:
76 void PutLE16(uint8_t* mem, int val) const {
77 mem[0] = (val >> 0) & 0xff;
78 mem[1] = (val >> 8) & 0xff;
80 void PutLE32(uint8_t* mem, int val) const {
81 mem[0] = (val >> 0) & 0xff;
82 mem[1] = (val >> 8) & 0xff;
83 mem[2] = (val >> 16) & 0xff;
84 mem[3] = (val >> 24) & 0xff;
88 uint32_t IVFWriter::WriteFileHeader(uint8_t* mem,
89 const std::string& codec,
90 int32_t width,
91 int32_t height) {
92 mem[0] = 'D';
93 mem[1] = 'K';
94 mem[2] = 'I';
95 mem[3] = 'F';
96 PutLE16(mem + 4, 0); // version
97 PutLE16(mem + 6, 32); // header size
98 PutLE32(mem + 8, fourcc(codec[0], codec[1], codec[2], '0')); // fourcc
99 PutLE16(mem + 12, static_cast<uint16_t>(width)); // width
100 PutLE16(mem + 14, static_cast<uint16_t>(height)); // height
101 PutLE32(mem + 16, 1000); // rate
102 PutLE32(mem + 20, 1); // scale
103 PutLE32(mem + 24, 0xffffffff); // length
104 PutLE32(mem + 28, 0); // unused
106 return 32;
109 uint32_t IVFWriter::WriteFrameHeader(uint8_t* mem,
110 uint64_t pts,
111 size_t frame_size) {
112 PutLE32(mem, (int)frame_size);
113 PutLE32(mem + 4, (int)(pts & 0xFFFFFFFF));
114 PutLE32(mem + 8, (int)(pts >> 32));
116 return 12;
119 // This object is the global object representing this plugin library as long
120 // as it is loaded.
121 class VideoEncoderModule : public pp::Module {
122 public:
123 VideoEncoderModule() : pp::Module() {}
124 virtual ~VideoEncoderModule() {}
126 virtual pp::Instance* CreateInstance(PP_Instance instance);
129 class VideoEncoderInstance : public pp::Instance {
130 public:
131 VideoEncoderInstance(PP_Instance instance, pp::Module* module);
132 virtual ~VideoEncoderInstance();
134 // pp::Instance implementation.
135 virtual void HandleMessage(const pp::Var& var_message);
137 private:
138 void AddVideoProfile(PP_VideoProfile profile, const std::string& profile_str);
139 void InitializeVideoProfiles();
140 PP_VideoProfile VideoProfileFromString(const std::string& str);
141 std::string VideoProfileToString(PP_VideoProfile profile);
143 void ConfigureTrack();
144 void OnConfiguredTrack(int32_t result);
145 void ProbeEncoder();
146 void OnEncoderProbed(int32_t result,
147 const std::vector<PP_VideoProfileDescription> profiles);
148 void StartEncoder();
149 void OnInitializedEncoder(int32_t result);
150 void ScheduleNextEncode();
151 void GetEncoderFrameTick(int32_t result);
152 void GetEncoderFrame(const pp::VideoFrame& track_frame);
153 void OnEncoderFrame(int32_t result,
154 pp::VideoFrame encoder_frame,
155 pp::VideoFrame track_frame);
156 int32_t CopyVideoFrame(pp::VideoFrame dest, pp::VideoFrame src);
157 void EncodeFrame(const pp::VideoFrame& frame);
158 void OnEncodeDone(int32_t result);
159 void OnGetBitstreamBuffer(int32_t result, PP_BitstreamBuffer buffer);
160 void StartTrackFrames();
161 void StopTrackFrames();
162 void OnTrackFrame(int32_t result, pp::VideoFrame frame);
164 void StopEncode();
166 void LogError(int32_t error, const std::string& message);
167 void Log(const std::string& message);
169 void PostDataMessage(const void* buffer, uint32_t size);
171 typedef std::map<std::string, PP_VideoProfile> VideoProfileFromStringMap;
172 VideoProfileFromStringMap profile_from_string_;
174 typedef std::map<PP_VideoProfile, std::string> VideoProfileToStringMap;
175 VideoProfileToStringMap profile_to_string_;
177 bool is_encoding_;
178 bool is_encode_ticking_;
179 bool is_receiving_track_frames_;
181 pp::VideoEncoder video_encoder_;
182 pp::MediaStreamVideoTrack video_track_;
183 pp::CompletionCallbackFactory<VideoEncoderInstance> callback_factory_;
185 PP_VideoProfile video_profile_;
186 PP_VideoFrame_Format frame_format_;
188 pp::Size requested_size_;
189 pp::Size frame_size_;
190 pp::Size encoder_size_;
191 uint32_t encoded_frames_;
193 std::deque<uint64_t> frames_timestamps_;
195 pp::VideoFrame current_track_frame_;
197 IVFWriter ivf_writer_;
199 PP_Time last_encode_tick_;
202 VideoEncoderInstance::VideoEncoderInstance(PP_Instance instance,
203 pp::Module* module)
204 : pp::Instance(instance),
205 is_encoding_(false),
206 is_encode_ticking_(false),
207 callback_factory_(this),
208 #if defined(USE_VP8_INSTEAD_OF_H264)
209 video_profile_(PP_VIDEOPROFILE_VP8_ANY),
210 #else
211 video_profile_(PP_VIDEOPROFILE_H264MAIN),
212 #endif
213 frame_format_(PP_VIDEOFRAME_FORMAT_I420),
214 encoded_frames_(0),
215 last_encode_tick_(0) {
216 InitializeVideoProfiles();
217 ProbeEncoder();
220 VideoEncoderInstance::~VideoEncoderInstance() {
223 void VideoEncoderInstance::AddVideoProfile(PP_VideoProfile profile,
224 const std::string& profile_str) {
225 profile_to_string_.insert(std::make_pair(profile, profile_str));
226 profile_from_string_.insert(std::make_pair(profile_str, profile));
229 void VideoEncoderInstance::InitializeVideoProfiles() {
230 AddVideoProfile(PP_VIDEOPROFILE_H264BASELINE, "h264baseline");
231 AddVideoProfile(PP_VIDEOPROFILE_H264MAIN, "h264main");
232 AddVideoProfile(PP_VIDEOPROFILE_H264EXTENDED, "h264extended");
233 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH, "h264high");
234 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH10PROFILE, "h264high10");
235 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH422PROFILE, "h264high422");
236 AddVideoProfile(PP_VIDEOPROFILE_H264HIGH444PREDICTIVEPROFILE,
237 "h264high444predictive");
238 AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEBASELINE, "h264scalablebaseline");
239 AddVideoProfile(PP_VIDEOPROFILE_H264SCALABLEHIGH, "h264scalablehigh");
240 AddVideoProfile(PP_VIDEOPROFILE_H264STEREOHIGH, "h264stereohigh");
241 AddVideoProfile(PP_VIDEOPROFILE_H264MULTIVIEWHIGH, "h264multiviewhigh");
242 AddVideoProfile(PP_VIDEOPROFILE_VP8_ANY, "vp8");
243 AddVideoProfile(PP_VIDEOPROFILE_VP9_ANY, "vp9");
246 PP_VideoProfile VideoEncoderInstance::VideoProfileFromString(
247 const std::string& str) {
248 VideoProfileFromStringMap::iterator it = profile_from_string_.find(str);
249 if (it == profile_from_string_.end())
250 return PP_VIDEOPROFILE_VP8_ANY;
251 return it->second;
254 std::string VideoEncoderInstance::VideoProfileToString(
255 PP_VideoProfile profile) {
256 VideoProfileToStringMap::iterator it = profile_to_string_.find(profile);
257 if (it == profile_to_string_.end())
258 return "unknown";
259 return it->second;
262 void VideoEncoderInstance::ConfigureTrack() {
263 if (encoder_size_.IsEmpty())
264 frame_size_ = requested_size_;
265 else
266 frame_size_ = encoder_size_;
268 int32_t attrib_list[] = {PP_MEDIASTREAMVIDEOTRACK_ATTRIB_FORMAT,
269 frame_format_,
270 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_WIDTH,
271 frame_size_.width(),
272 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_HEIGHT,
273 frame_size_.height(),
274 PP_MEDIASTREAMVIDEOTRACK_ATTRIB_NONE};
276 video_track_.Configure(
277 attrib_list,
278 callback_factory_.NewCallback(&VideoEncoderInstance::OnConfiguredTrack));
281 void VideoEncoderInstance::OnConfiguredTrack(int32_t result) {
282 if (result != PP_OK) {
283 LogError(result, "Cannot configure track");
284 return;
287 if (is_encoding_) {
288 StartTrackFrames();
289 ScheduleNextEncode();
290 } else
291 StartEncoder();
294 void VideoEncoderInstance::ProbeEncoder() {
295 video_encoder_ = pp::VideoEncoder(this);
296 video_encoder_.GetSupportedProfiles(callback_factory_.NewCallbackWithOutput(
297 &VideoEncoderInstance::OnEncoderProbed));
300 void VideoEncoderInstance::OnEncoderProbed(
301 int32_t result,
302 const std::vector<PP_VideoProfileDescription> profiles) {
303 pp::VarDictionary dict;
304 dict.Set(pp::Var("name"), pp::Var("supportedProfiles"));
305 pp::VarArray js_profiles;
306 dict.Set(pp::Var("profiles"), js_profiles);
308 if (result < 0) {
309 LogError(result, "Cannot get supported profiles");
310 PostMessage(dict);
313 int32_t idx = 0;
314 for (uint32_t i = 0; i < profiles.size(); i++) {
315 const PP_VideoProfileDescription& profile = profiles[i];
316 js_profiles.Set(idx++, pp::Var(VideoProfileToString(profile.profile)));
318 PostMessage(dict);
321 void VideoEncoderInstance::StartEncoder() {
322 video_encoder_ = pp::VideoEncoder(this);
323 frames_timestamps_.clear();
325 int32_t error = video_encoder_.Initialize(
326 frame_format_, frame_size_, video_profile_, 2000000,
327 PP_HARDWAREACCELERATION_WITHFALLBACK,
328 callback_factory_.NewCallback(
329 &VideoEncoderInstance::OnInitializedEncoder));
330 if (error != PP_OK_COMPLETIONPENDING) {
331 LogError(error, "Cannot initialize encoder");
332 return;
336 void VideoEncoderInstance::OnInitializedEncoder(int32_t result) {
337 if (result != PP_OK) {
338 LogError(result, "Encoder initialization failed");
339 return;
342 is_encoding_ = true;
343 Log("started");
345 if (video_encoder_.GetFrameCodedSize(&encoder_size_) != PP_OK) {
346 LogError(result, "Cannot get encoder coded frame size");
347 return;
350 video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
351 &VideoEncoderInstance::OnGetBitstreamBuffer));
353 if (encoder_size_ != frame_size_)
354 ConfigureTrack();
355 else {
356 StartTrackFrames();
357 ScheduleNextEncode();
361 void VideoEncoderInstance::ScheduleNextEncode() {
362 // Avoid scheduling more than once at a time.
363 if (is_encode_ticking_)
364 return;
366 PP_Time now = pp::Module::Get()->core()->GetTime();
367 PP_Time tick = 1.0 / 30;
368 // If the callback was triggered late, we need to account for that
369 // delay for the next tick.
370 PP_Time delta = tick - clamp(0, tick, now - last_encode_tick_ - tick);
372 pp::Module::Get()->core()->CallOnMainThread(
373 delta * 1000,
374 callback_factory_.NewCallback(&VideoEncoderInstance::GetEncoderFrameTick),
377 last_encode_tick_ = now;
378 is_encode_ticking_ = true;
381 void VideoEncoderInstance::GetEncoderFrameTick(int32_t result) {
382 is_encode_ticking_ = false;
384 if (is_encoding_) {
385 if (!current_track_frame_.is_null()) {
386 pp::VideoFrame frame = current_track_frame_;
387 current_track_frame_.detach();
388 GetEncoderFrame(frame);
390 ScheduleNextEncode();
394 void VideoEncoderInstance::GetEncoderFrame(const pp::VideoFrame& track_frame) {
395 video_encoder_.GetVideoFrame(callback_factory_.NewCallbackWithOutput(
396 &VideoEncoderInstance::OnEncoderFrame, track_frame));
399 void VideoEncoderInstance::OnEncoderFrame(int32_t result,
400 pp::VideoFrame encoder_frame,
401 pp::VideoFrame track_frame) {
402 if (result == PP_ERROR_ABORTED) {
403 video_track_.RecycleFrame(track_frame);
404 return;
406 if (result != PP_OK) {
407 video_track_.RecycleFrame(track_frame);
408 LogError(result, "Cannot get video frame from video encoder");
409 return;
412 track_frame.GetSize(&frame_size_);
414 if (frame_size_ != encoder_size_) {
415 video_track_.RecycleFrame(track_frame);
416 LogError(PP_ERROR_FAILED, "MediaStreamVideoTrack frame size incorrect");
417 return;
420 if (CopyVideoFrame(encoder_frame, track_frame) == PP_OK)
421 EncodeFrame(encoder_frame);
422 video_track_.RecycleFrame(track_frame);
425 int32_t VideoEncoderInstance::CopyVideoFrame(pp::VideoFrame dest,
426 pp::VideoFrame src) {
427 if (dest.GetDataBufferSize() < src.GetDataBufferSize()) {
428 std::ostringstream oss;
429 oss << "Incorrect destination video frame buffer size : "
430 << dest.GetDataBufferSize() << " < " << src.GetDataBufferSize();
431 LogError(PP_ERROR_FAILED, oss.str());
432 return PP_ERROR_FAILED;
435 dest.SetTimestamp(src.GetTimestamp());
436 memcpy(dest.GetDataBuffer(), src.GetDataBuffer(), src.GetDataBufferSize());
437 return PP_OK;
440 void VideoEncoderInstance::EncodeFrame(const pp::VideoFrame& frame) {
441 frames_timestamps_.push_back(
442 static_cast<uint64_t>(frame.GetTimestamp() * 1000));
443 video_encoder_.Encode(
444 frame, PP_FALSE,
445 callback_factory_.NewCallback(&VideoEncoderInstance::OnEncodeDone));
448 void VideoEncoderInstance::OnEncodeDone(int32_t result) {
449 if (result == PP_ERROR_ABORTED)
450 return;
451 if (result != PP_OK)
452 LogError(result, "Encode failed");
455 void VideoEncoderInstance::OnGetBitstreamBuffer(int32_t result,
456 PP_BitstreamBuffer buffer) {
457 if (result == PP_ERROR_ABORTED)
458 return;
459 if (result != PP_OK) {
460 LogError(result, "Cannot get bitstream buffer");
461 return;
464 encoded_frames_++;
465 PostDataMessage(buffer.buffer, buffer.size);
466 video_encoder_.RecycleBitstreamBuffer(buffer);
468 video_encoder_.GetBitstreamBuffer(callback_factory_.NewCallbackWithOutput(
469 &VideoEncoderInstance::OnGetBitstreamBuffer));
472 void VideoEncoderInstance::StartTrackFrames() {
473 is_receiving_track_frames_ = true;
474 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
475 &VideoEncoderInstance::OnTrackFrame));
478 void VideoEncoderInstance::StopTrackFrames() {
479 is_receiving_track_frames_ = false;
480 if (!current_track_frame_.is_null()) {
481 video_track_.RecycleFrame(current_track_frame_);
482 current_track_frame_.detach();
486 void VideoEncoderInstance::OnTrackFrame(int32_t result, pp::VideoFrame frame) {
487 if (result == PP_ERROR_ABORTED)
488 return;
490 if (!current_track_frame_.is_null()) {
491 video_track_.RecycleFrame(current_track_frame_);
492 current_track_frame_.detach();
495 if (result != PP_OK) {
496 LogError(result, "Cannot get video frame from video track");
497 return;
500 current_track_frame_ = frame;
501 if (is_receiving_track_frames_)
502 video_track_.GetFrame(callback_factory_.NewCallbackWithOutput(
503 &VideoEncoderInstance::OnTrackFrame));
506 void VideoEncoderInstance::StopEncode() {
507 video_encoder_.Close();
508 StopTrackFrames();
509 video_track_.Close();
510 is_encoding_ = false;
511 encoded_frames_ = 0;
516 void VideoEncoderInstance::HandleMessage(const pp::Var& var_message) {
517 if (!var_message.is_dictionary()) {
518 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid message!"));
519 return;
522 pp::VarDictionary dict_message(var_message);
523 std::string command = dict_message.Get("command").AsString();
525 if (command == "start") {
526 requested_size_ = pp::Size(dict_message.Get("width").AsInt(),
527 dict_message.Get("height").AsInt());
528 pp::Var var_track = dict_message.Get("track");
529 if (!var_track.is_resource()) {
530 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Given track is not a resource"));
531 return;
533 pp::Resource resource_track = var_track.AsResource();
534 video_track_ = pp::MediaStreamVideoTrack(resource_track);
535 video_encoder_ = pp::VideoEncoder();
536 video_profile_ = VideoProfileFromString(
537 dict_message.Get("profile").AsString());
538 ConfigureTrack();
539 } else if (command == "stop") {
540 StopEncode();
541 Log("stopped");
542 } else {
543 LogToConsole(PP_LOGLEVEL_ERROR, pp::Var("Invalid command!"));
547 void VideoEncoderInstance::PostDataMessage(const void* buffer, uint32_t size) {
548 pp::VarDictionary dictionary;
550 dictionary.Set(pp::Var("name"), pp::Var("data"));
552 pp::VarArrayBuffer array_buffer;
553 uint8_t* data_ptr;
554 uint32_t data_offset = 0;
555 if (video_profile_ == PP_VIDEOPROFILE_VP8_ANY ||
556 video_profile_ == PP_VIDEOPROFILE_VP9_ANY) {
557 uint32_t frame_offset = 0;
558 if (encoded_frames_ == 1) {
559 array_buffer = pp::VarArrayBuffer(
560 size + ivf_writer_.GetFileHeaderSize() +
561 ivf_writer_.GetFrameHeaderSize());
562 data_ptr = static_cast<uint8_t*>(array_buffer.Map());
563 frame_offset = ivf_writer_.WriteFileHeader(
564 data_ptr, ToUpperString(VideoProfileToString(video_profile_)),
565 frame_size_.width(), frame_size_.height());
566 } else {
567 array_buffer = pp::VarArrayBuffer(
568 size + ivf_writer_.GetFrameHeaderSize());
569 data_ptr = static_cast<uint8_t*>(array_buffer.Map());
571 uint64_t timestamp = frames_timestamps_.front();
572 frames_timestamps_.pop_front();
573 data_offset =
574 frame_offset +
575 ivf_writer_.WriteFrameHeader(data_ptr + frame_offset, timestamp, size);
576 } else {
577 array_buffer = pp::VarArrayBuffer(size);
578 data_ptr = static_cast<uint8_t*>(array_buffer.Map());
581 memcpy(data_ptr + data_offset, buffer, size);
582 array_buffer.Unmap();
583 dictionary.Set(pp::Var("data"), array_buffer);
585 PostMessage(dictionary);
588 void VideoEncoderInstance::LogError(int32_t error, const std::string& message) {
589 std::string msg("Error: ");
590 msg.append(pp::Var(error).DebugString());
591 msg.append(" : ");
592 msg.append(message);
594 Log(msg);
597 void VideoEncoderInstance::Log(const std::string& message) {
598 pp::VarDictionary dictionary;
599 dictionary.Set(pp::Var("name"), pp::Var("log"));
600 dictionary.Set(pp::Var("message"), pp::Var(message));
602 PostMessage(dictionary);
605 pp::Instance* VideoEncoderModule::CreateInstance(PP_Instance instance) {
606 return new VideoEncoderInstance(instance, this);
609 } // anonymous namespace
611 namespace pp {
612 // Factory function for your specialization of the Module object.
613 Module* CreateModule() {
614 return new VideoEncoderModule();
616 } // namespace pp