Revert of Fix missing GN dependencies. (patchset #4 id:60001 of https://codereview...
[chromium-blink-merge.git] / remoting / codec / video_encoder_vpx.cc
blobc74f0d8ecf5e62c4c9d79d77127fb3408ff84932
1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "remoting/codec/video_encoder_vpx.h"
7 #include "base/bind.h"
8 #include "base/command_line.h"
9 #include "base/logging.h"
10 #include "base/sys_info.h"
11 #include "remoting/base/util.h"
12 #include "remoting/proto/video.pb.h"
13 #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
14 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
15 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
16 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
18 extern "C" {
19 #define VPX_CODEC_DISABLE_COMPAT 1
20 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
24 namespace remoting {
26 namespace {
28 // Name of command-line flag to enable VP9 to use I444 by default.
29 const char kEnableI444SwitchName[] = "enable-i444";
31 // Number of bytes in an RGBx pixel.
32 const int kBytesPerRgbPixel = 4;
34 // Defines the dimension of a macro block. This is used to compute the active
35 // map for the encoder.
36 const int kMacroBlockSize = 16;
38 // Magic encoder profile numbers for I420 and I444 input formats.
39 const int kVp9I420ProfileNumber = 0;
40 const int kVp9I444ProfileNumber = 1;
42 // Magic encoder constants for adaptive quantization strategy.
43 const int kVp9AqModeNone = 0;
44 const int kVp9AqModeCyclicRefresh = 3;
46 void SetCommonCodecParameters(vpx_codec_enc_cfg_t* config,
47 const webrtc::DesktopSize& size) {
48 // Use millisecond granularity time base.
49 config->g_timebase.num = 1;
50 config->g_timebase.den = 1000;
52 config->g_w = size.width();
53 config->g_h = size.height();
54 config->g_pass = VPX_RC_ONE_PASS;
56 // Start emitting packets immediately.
57 config->g_lag_in_frames = 0;
59 // Since the transport layer is reliable, keyframes should not be necessary.
60 // However, due to crbug.com/440223, decoding fails after 30,000 non-key
61 // frames, so take the hit of an "unnecessary" key-frame every 10,000 frames.
62 config->kf_min_dist = 10000;
63 config->kf_max_dist = 10000;
65 // Using 2 threads gives a great boost in performance for most systems with
66 // adequate processing power. NB: Going to multiple threads on low end
67 // windows systems can really hurt performance.
68 // http://crbug.com/99179
69 config->g_threads = (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1;
72 void SetVp8CodecParameters(vpx_codec_enc_cfg_t* config,
73 const webrtc::DesktopSize& size) {
74 // Adjust default target bit-rate to account for actual desktop size.
75 config->rc_target_bitrate = size.width() * size.height() *
76 config->rc_target_bitrate / config->g_w / config->g_h;
78 SetCommonCodecParameters(config, size);
80 // Value of 2 means using the real time profile. This is basically a
81 // redundant option since we explicitly select real time mode when doing
82 // encoding.
83 config->g_profile = 2;
85 // Clamping the quantizer constrains the worst-case quality and CPU usage.
86 config->rc_min_quantizer = 20;
87 config->rc_max_quantizer = 30;
90 void SetVp9CodecParameters(vpx_codec_enc_cfg_t* config,
91 const webrtc::DesktopSize& size,
92 bool lossless_color,
93 bool lossless_encode) {
94 SetCommonCodecParameters(config, size);
96 // Configure VP9 for I420 or I444 source frames.
97 config->g_profile =
98 lossless_color ? kVp9I444ProfileNumber : kVp9I420ProfileNumber;
100 if (lossless_encode) {
101 // Disable quantization entirely, putting the encoder in "lossless" mode.
102 config->rc_min_quantizer = 0;
103 config->rc_max_quantizer = 0;
104 config->rc_end_usage = VPX_VBR;
105 } else {
106 config->rc_min_quantizer = 4;
107 config->rc_max_quantizer = 30;
108 config->rc_end_usage = VPX_CBR;
109 // In the absence of a good bandwidth estimator set the target bitrate to a
110 // conservative default.
111 config->rc_target_bitrate = 500;
115 void SetVp8CodecOptions(vpx_codec_ctx_t* codec) {
116 // CPUUSED of 16 will have the smallest CPU load. This turns off sub-pixel
117 // motion search.
118 vpx_codec_err_t ret = vpx_codec_control(codec, VP8E_SET_CPUUSED, 16);
119 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set CPUUSED";
121 // Use the lowest level of noise sensitivity so as to spend less time
122 // on motion estimation and inter-prediction mode.
123 ret = vpx_codec_control(codec, VP8E_SET_NOISE_SENSITIVITY, 0);
124 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set noise sensitivity";
127 void SetVp9CodecOptions(vpx_codec_ctx_t* codec, bool lossless_encode) {
128 // Request the lowest-CPU usage that VP9 supports, which depends on whether
129 // we are encoding lossy or lossless.
130 // Note that this is configured via the same parameter as for VP8.
131 int cpu_used = lossless_encode ? 5 : 6;
132 vpx_codec_err_t ret = vpx_codec_control(codec, VP8E_SET_CPUUSED, cpu_used);
133 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set CPUUSED";
135 // Use the lowest level of noise sensitivity so as to spend less time
136 // on motion estimation and inter-prediction mode.
137 ret = vpx_codec_control(codec, VP9E_SET_NOISE_SENSITIVITY, 0);
138 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set noise sensitivity";
140 // Configure the codec to tune it for screen media.
141 ret = vpx_codec_control(
142 codec, VP9E_SET_TUNE_CONTENT, VP9E_CONTENT_SCREEN);
143 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set screen content mode";
145 // Set cyclic refresh (aka "top-off") only for lossy encoding.
146 int aq_mode = lossless_encode ? kVp9AqModeNone : kVp9AqModeCyclicRefresh;
147 ret = vpx_codec_control(codec, VP9E_SET_AQ_MODE, aq_mode);
148 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set aq mode";
151 void FreeImageIfMismatched(bool use_i444,
152 const webrtc::DesktopSize& size,
153 scoped_ptr<vpx_image_t>* out_image,
154 scoped_ptr<uint8[]>* out_image_buffer) {
155 if (*out_image) {
156 const vpx_img_fmt_t desired_fmt =
157 use_i444 ? VPX_IMG_FMT_I444 : VPX_IMG_FMT_I420;
158 if (!size.equals(webrtc::DesktopSize((*out_image)->w, (*out_image)->h)) ||
159 (*out_image)->fmt != desired_fmt) {
160 out_image_buffer->reset();
161 out_image->reset();
166 void CreateImage(bool use_i444,
167 const webrtc::DesktopSize& size,
168 scoped_ptr<vpx_image_t>* out_image,
169 scoped_ptr<uint8[]>* out_image_buffer) {
170 DCHECK(!size.is_empty());
171 DCHECK(!*out_image_buffer);
172 DCHECK(!*out_image);
174 scoped_ptr<vpx_image_t> image(new vpx_image_t());
175 memset(image.get(), 0, sizeof(vpx_image_t));
177 // libvpx seems to require both to be assigned.
178 image->d_w = size.width();
179 image->w = size.width();
180 image->d_h = size.height();
181 image->h = size.height();
183 // libvpx should derive chroma shifts from|fmt| but currently has a bug:
184 // https://code.google.com/p/webm/issues/detail?id=627
185 if (use_i444) {
186 image->fmt = VPX_IMG_FMT_I444;
187 image->x_chroma_shift = 0;
188 image->y_chroma_shift = 0;
189 } else { // I420
190 image->fmt = VPX_IMG_FMT_YV12;
191 image->x_chroma_shift = 1;
192 image->y_chroma_shift = 1;
195 // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad
196 // the Y, U and V planes' strides to multiples of 16 bytes.
197 const int y_stride = ((image->w - 1) & ~15) + 16;
198 const int uv_unaligned_stride = y_stride >> image->x_chroma_shift;
199 const int uv_stride = ((uv_unaligned_stride - 1) & ~15) + 16;
201 // libvpx accesses the source image in macro blocks, and will over-read
202 // if the image is not padded out to the next macroblock: crbug.com/119633.
203 // Pad the Y, U and V planes' height out to compensate.
204 // Assuming macroblocks are 16x16, aligning the planes' strides above also
205 // macroblock aligned them.
206 static_assert(kMacroBlockSize == 16, "macroblock_size_not_16");
207 const int y_rows = ((image->h - 1) & ~(kMacroBlockSize-1)) + kMacroBlockSize;
208 const int uv_rows = y_rows >> image->y_chroma_shift;
210 // Allocate a YUV buffer large enough for the aligned data & padding.
211 const int buffer_size = y_stride * y_rows + 2*uv_stride * uv_rows;
212 scoped_ptr<uint8[]> image_buffer(new uint8[buffer_size]);
214 // Reset image value to 128 so we just need to fill in the y plane.
215 memset(image_buffer.get(), 128, buffer_size);
217 // Fill in the information for |image_|.
218 unsigned char* uchar_buffer =
219 reinterpret_cast<unsigned char*>(image_buffer.get());
220 image->planes[0] = uchar_buffer;
221 image->planes[1] = image->planes[0] + y_stride * y_rows;
222 image->planes[2] = image->planes[1] + uv_stride * uv_rows;
223 image->stride[0] = y_stride;
224 image->stride[1] = uv_stride;
225 image->stride[2] = uv_stride;
227 *out_image = image.Pass();
228 *out_image_buffer = image_buffer.Pass();
231 } // namespace
233 // static
234 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP8() {
235 return make_scoped_ptr(new VideoEncoderVpx(false));
238 // static
239 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP9() {
240 return make_scoped_ptr(new VideoEncoderVpx(true));
243 VideoEncoderVpx::~VideoEncoderVpx() {}
245 void VideoEncoderVpx::SetLosslessEncode(bool want_lossless) {
246 if (use_vp9_ && (want_lossless != lossless_encode_)) {
247 lossless_encode_ = want_lossless;
248 if (codec_)
249 Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
250 codec_->config.enc->g_h));
254 void VideoEncoderVpx::SetLosslessColor(bool want_lossless) {
255 if (use_vp9_ && (want_lossless != lossless_color_)) {
256 lossless_color_ = want_lossless;
257 // TODO(wez): Switch to ConfigureCodec() path once libvpx supports it.
258 // See https://code.google.com/p/webm/issues/detail?id=913.
259 //if (codec_)
260 // Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
261 // codec_->config.enc->g_h));
262 codec_.reset();
266 scoped_ptr<VideoPacket> VideoEncoderVpx::Encode(
267 const webrtc::DesktopFrame& frame) {
268 DCHECK_LE(32, frame.size().width());
269 DCHECK_LE(32, frame.size().height());
271 base::TimeTicks encode_start_time = base::TimeTicks::Now();
273 // Create or reconfigure the codec to match the size of |frame|.
274 if (!codec_ ||
275 (image_ &&
276 !frame.size().equals(webrtc::DesktopSize(image_->w, image_->h)))) {
277 Configure(frame.size());
280 // Convert the updated capture data ready for encode.
281 webrtc::DesktopRegion updated_region;
282 PrepareImage(frame, &updated_region);
284 // Update active map based on updated region.
285 SetActiveMapFromRegion(updated_region);
287 // Apply active map to the encoder.
288 vpx_active_map_t act_map;
289 act_map.rows = active_map_height_;
290 act_map.cols = active_map_width_;
291 act_map.active_map = active_map_.get();
292 if (vpx_codec_control(codec_.get(), VP8E_SET_ACTIVEMAP, &act_map)) {
293 LOG(ERROR) << "Unable to apply active map";
296 // Do the actual encoding.
297 int timestamp = (encode_start_time - timestamp_base_).InMilliseconds();
298 vpx_codec_err_t ret = vpx_codec_encode(
299 codec_.get(), image_.get(), timestamp, 1, 0, VPX_DL_REALTIME);
300 DCHECK_EQ(ret, VPX_CODEC_OK)
301 << "Encoding error: " << vpx_codec_err_to_string(ret) << "\n"
302 << "Details: " << vpx_codec_error(codec_.get()) << "\n"
303 << vpx_codec_error_detail(codec_.get());
305 if (use_vp9_ && !lossless_encode_) {
306 ret = vpx_codec_control(codec_.get(), VP9E_GET_ACTIVEMAP, &act_map);
307 DCHECK_EQ(ret, VPX_CODEC_OK)
308 << "Failed to fetch active map: "
309 << vpx_codec_err_to_string(ret) << "\n";
310 UpdateRegionFromActiveMap(&updated_region);
313 // Read the encoded data.
314 vpx_codec_iter_t iter = NULL;
315 bool got_data = false;
317 // TODO(hclam): Make sure we get exactly one frame from the packet.
318 // TODO(hclam): We should provide the output buffer to avoid one copy.
319 scoped_ptr<VideoPacket> packet(
320 helper_.CreateVideoPacketWithUpdatedRegion(frame, updated_region));
321 packet->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8);
323 while (!got_data) {
324 const vpx_codec_cx_pkt_t* vpx_packet =
325 vpx_codec_get_cx_data(codec_.get(), &iter);
326 if (!vpx_packet)
327 continue;
329 switch (vpx_packet->kind) {
330 case VPX_CODEC_CX_FRAME_PKT:
331 got_data = true;
332 packet->set_data(vpx_packet->data.frame.buf, vpx_packet->data.frame.sz);
333 break;
334 default:
335 break;
339 // Note the time taken to encode the pixel data.
340 packet->set_encode_time_ms(
341 (base::TimeTicks::Now() - encode_start_time).InMillisecondsRoundedUp());
343 return packet.Pass();
346 VideoEncoderVpx::VideoEncoderVpx(bool use_vp9)
347 : use_vp9_(use_vp9),
348 lossless_encode_(false),
349 lossless_color_(false),
350 active_map_width_(0),
351 active_map_height_(0) {
352 if (use_vp9_) {
353 // Use I444 colour space, by default, if specified on the command-line.
354 if (base::CommandLine::ForCurrentProcess()->HasSwitch(
355 kEnableI444SwitchName)) {
356 SetLosslessColor(true);
361 void VideoEncoderVpx::Configure(const webrtc::DesktopSize& size) {
362 DCHECK(use_vp9_ || !lossless_color_);
363 DCHECK(use_vp9_ || !lossless_encode_);
365 // Tear down |image_| if it no longer matches the size and color settings.
366 // PrepareImage() will then create a new buffer of the required dimensions if
367 // |image_| is not allocated.
368 FreeImageIfMismatched(lossless_color_, size, &image_, &image_buffer_);
370 // Initialize active map.
371 active_map_width_ = (size.width() + kMacroBlockSize - 1) / kMacroBlockSize;
372 active_map_height_ = (size.height() + kMacroBlockSize - 1) / kMacroBlockSize;
373 active_map_.reset(new uint8[active_map_width_ * active_map_height_]);
375 // TODO(wez): Remove this hack once VPX can handle frame size reconfiguration.
376 // See https://code.google.com/p/webm/issues/detail?id=912.
377 if (codec_) {
378 // If the frame size has changed then force re-creation of the codec.
379 if (codec_->config.enc->g_w != static_cast<unsigned int>(size.width()) ||
380 codec_->config.enc->g_h != static_cast<unsigned int>(size.height())) {
381 codec_.reset();
385 // (Re)Set the base for frame timestamps if the codec is being (re)created.
386 if (!codec_) {
387 timestamp_base_ = base::TimeTicks::Now();
390 // Fetch a default configuration for the desired codec.
391 const vpx_codec_iface_t* interface =
392 use_vp9_ ? vpx_codec_vp9_cx() : vpx_codec_vp8_cx();
393 vpx_codec_enc_cfg_t config;
394 vpx_codec_err_t ret = vpx_codec_enc_config_default(interface, &config, 0);
395 DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to fetch default configuration";
397 // Customize the default configuration to our needs.
398 if (use_vp9_) {
399 SetVp9CodecParameters(&config, size, lossless_color_, lossless_encode_);
400 } else {
401 SetVp8CodecParameters(&config, size);
404 // Initialize or re-configure the codec with the custom configuration.
405 if (!codec_) {
406 codec_.reset(new vpx_codec_ctx_t);
407 ret = vpx_codec_enc_init(codec_.get(), interface, &config, 0);
408 CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to initialize codec";
409 } else {
410 ret = vpx_codec_enc_config_set(codec_.get(), &config);
411 CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to reconfigure codec";
414 // Apply further customizations to the codec now it's initialized.
415 if (use_vp9_) {
416 SetVp9CodecOptions(codec_.get(), lossless_encode_);
417 } else {
418 SetVp8CodecOptions(codec_.get());
422 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame& frame,
423 webrtc::DesktopRegion* updated_region) {
424 if (frame.updated_region().is_empty()) {
425 updated_region->Clear();
426 return;
429 updated_region->Clear();
430 if (image_) {
431 // Pad each rectangle to avoid the block-artefact filters in libvpx from
432 // introducing artefacts; VP9 includes up to 8px either side, and VP8 up to
433 // 3px, so unchanged pixels up to that far out may still be affected by the
434 // changes in the updated region, and so must be listed in the active map.
435 // After padding we align each rectangle to 16x16 active-map macroblocks.
436 // This implicitly ensures all rects have even top-left coords, which is
437 // is required by ConvertRGBToYUVWithRect().
438 // TODO(wez): Do we still need 16x16 align, or is even alignment sufficient?
439 int padding = use_vp9_ ? 8 : 3;
440 for (webrtc::DesktopRegion::Iterator r(frame.updated_region());
441 !r.IsAtEnd(); r.Advance()) {
442 const webrtc::DesktopRect& rect = r.rect();
443 updated_region->AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
444 rect.left() - padding, rect.top() - padding, rect.right() + padding,
445 rect.bottom() + padding)));
447 DCHECK(!updated_region->is_empty());
449 // Clip back to the screen dimensions, in case they're not macroblock
450 // aligned. The conversion routines don't require even width & height,
451 // so this is safe even if the source dimensions are not even.
452 updated_region->IntersectWith(
453 webrtc::DesktopRect::MakeWH(image_->w, image_->h));
454 } else {
455 CreateImage(lossless_color_, frame.size(), &image_, &image_buffer_);
456 updated_region->AddRect(webrtc::DesktopRect::MakeWH(image_->w, image_->h));
459 // Convert the updated region to YUV ready for encoding.
460 const uint8* rgb_data = frame.data();
461 const int rgb_stride = frame.stride();
462 const int y_stride = image_->stride[0];
463 DCHECK_EQ(image_->stride[1], image_->stride[2]);
464 const int uv_stride = image_->stride[1];
465 uint8* y_data = image_->planes[0];
466 uint8* u_data = image_->planes[1];
467 uint8* v_data = image_->planes[2];
469 switch (image_->fmt) {
470 case VPX_IMG_FMT_I444:
471 for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
472 r.Advance()) {
473 const webrtc::DesktopRect& rect = r.rect();
474 int rgb_offset = rgb_stride * rect.top() +
475 rect.left() * kBytesPerRgbPixel;
476 int yuv_offset = uv_stride * rect.top() + rect.left();
477 libyuv::ARGBToI444(rgb_data + rgb_offset, rgb_stride,
478 y_data + yuv_offset, y_stride,
479 u_data + yuv_offset, uv_stride,
480 v_data + yuv_offset, uv_stride,
481 rect.width(), rect.height());
483 break;
484 case VPX_IMG_FMT_YV12:
485 for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
486 r.Advance()) {
487 const webrtc::DesktopRect& rect = r.rect();
488 int rgb_offset = rgb_stride * rect.top() +
489 rect.left() * kBytesPerRgbPixel;
490 int y_offset = y_stride * rect.top() + rect.left();
491 int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2;
492 libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride,
493 y_data + y_offset, y_stride,
494 u_data + uv_offset, uv_stride,
495 v_data + uv_offset, uv_stride,
496 rect.width(), rect.height());
498 break;
499 default:
500 NOTREACHED();
501 break;
505 void VideoEncoderVpx::SetActiveMapFromRegion(
506 const webrtc::DesktopRegion& updated_region) {
507 // Clear active map first.
508 memset(active_map_.get(), 0, active_map_width_ * active_map_height_);
510 // Mark updated areas active.
511 for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
512 r.Advance()) {
513 const webrtc::DesktopRect& rect = r.rect();
514 int left = rect.left() / kMacroBlockSize;
515 int right = (rect.right() - 1) / kMacroBlockSize;
516 int top = rect.top() / kMacroBlockSize;
517 int bottom = (rect.bottom() - 1) / kMacroBlockSize;
518 DCHECK_LT(right, active_map_width_);
519 DCHECK_LT(bottom, active_map_height_);
521 uint8* map = active_map_.get() + top * active_map_width_;
522 for (int y = top; y <= bottom; ++y) {
523 for (int x = left; x <= right; ++x)
524 map[x] = 1;
525 map += active_map_width_;
530 void VideoEncoderVpx::UpdateRegionFromActiveMap(
531 webrtc::DesktopRegion* updated_region) {
532 const uint8* map = active_map_.get();
533 for (int y = 0; y < active_map_height_; ++y) {
534 for (int x0 = 0; x0 < active_map_width_;) {
535 int x1 = x0;
536 for (; x1 < active_map_width_; ++x1) {
537 if (map[y * active_map_width_ + x1] == 0)
538 break;
540 if (x1 > x0) {
541 updated_region->AddRect(webrtc::DesktopRect::MakeLTRB(
542 kMacroBlockSize * x0, kMacroBlockSize * y, kMacroBlockSize * x1,
543 kMacroBlockSize * (y + 1)));
545 x0 = x1 + 1;
548 updated_region->IntersectWith(
549 webrtc::DesktopRect::MakeWH(image_->w, image_->h));
552 } // namespace remoting