1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "remoting/codec/video_encoder_vpx.h"
8 #include "base/command_line.h"
9 #include "base/logging.h"
10 #include "base/sys_info.h"
11 #include "remoting/base/util.h"
12 #include "remoting/proto/video.pb.h"
13 #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
14 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
15 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
16 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
19 #define VPX_CODEC_DISABLE_COMPAT 1
20 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
28 // Name of command-line flag to enable VP9 to use I444 by default.
29 const char kEnableI444SwitchName
[] = "enable-i444";
31 // Number of bytes in an RGBx pixel.
32 const int kBytesPerRgbPixel
= 4;
34 // Defines the dimension of a macro block. This is used to compute the active
35 // map for the encoder.
36 const int kMacroBlockSize
= 16;
38 // Magic encoder profile numbers for I420 and I444 input formats.
39 const int kVp9I420ProfileNumber
= 0;
40 const int kVp9I444ProfileNumber
= 1;
42 // Magic encoder constants for adaptive quantization strategy.
43 const int kVp9AqModeNone
= 0;
44 const int kVp9AqModeCyclicRefresh
= 3;
46 void SetCommonCodecParameters(vpx_codec_enc_cfg_t
* config
,
47 const webrtc::DesktopSize
& size
) {
48 // Use millisecond granularity time base.
49 config
->g_timebase
.num
= 1;
50 config
->g_timebase
.den
= 1000;
52 config
->g_w
= size
.width();
53 config
->g_h
= size
.height();
54 config
->g_pass
= VPX_RC_ONE_PASS
;
56 // Start emitting packets immediately.
57 config
->g_lag_in_frames
= 0;
59 // Since the transport layer is reliable, keyframes should not be necessary.
60 // However, due to crbug.com/440223, decoding fails after 30,000 non-key
61 // frames, so take the hit of an "unnecessary" key-frame every 10,000 frames.
62 config
->kf_min_dist
= 10000;
63 config
->kf_max_dist
= 10000;
65 // Using 2 threads gives a great boost in performance for most systems with
66 // adequate processing power. NB: Going to multiple threads on low end
67 // windows systems can really hurt performance.
68 // http://crbug.com/99179
69 config
->g_threads
= (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1;
72 void SetVp8CodecParameters(vpx_codec_enc_cfg_t
* config
,
73 const webrtc::DesktopSize
& size
) {
74 // Adjust default target bit-rate to account for actual desktop size.
75 config
->rc_target_bitrate
= size
.width() * size
.height() *
76 config
->rc_target_bitrate
/ config
->g_w
/ config
->g_h
;
78 SetCommonCodecParameters(config
, size
);
80 // Value of 2 means using the real time profile. This is basically a
81 // redundant option since we explicitly select real time mode when doing
83 config
->g_profile
= 2;
85 // Clamping the quantizer constrains the worst-case quality and CPU usage.
86 config
->rc_min_quantizer
= 20;
87 config
->rc_max_quantizer
= 30;
90 void SetVp9CodecParameters(vpx_codec_enc_cfg_t
* config
,
91 const webrtc::DesktopSize
& size
,
93 bool lossless_encode
) {
94 SetCommonCodecParameters(config
, size
);
96 // Configure VP9 for I420 or I444 source frames.
98 lossless_color
? kVp9I444ProfileNumber
: kVp9I420ProfileNumber
;
100 if (lossless_encode
) {
101 // Disable quantization entirely, putting the encoder in "lossless" mode.
102 config
->rc_min_quantizer
= 0;
103 config
->rc_max_quantizer
= 0;
104 config
->rc_end_usage
= VPX_VBR
;
106 config
->rc_min_quantizer
= 4;
107 config
->rc_max_quantizer
= 30;
108 config
->rc_end_usage
= VPX_CBR
;
109 // In the absence of a good bandwidth estimator set the target bitrate to a
110 // conservative default.
111 config
->rc_target_bitrate
= 500;
115 void SetVp8CodecOptions(vpx_codec_ctx_t
* codec
) {
116 // CPUUSED of 16 will have the smallest CPU load. This turns off sub-pixel
118 vpx_codec_err_t ret
= vpx_codec_control(codec
, VP8E_SET_CPUUSED
, 16);
119 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set CPUUSED";
121 // Use the lowest level of noise sensitivity so as to spend less time
122 // on motion estimation and inter-prediction mode.
123 ret
= vpx_codec_control(codec
, VP8E_SET_NOISE_SENSITIVITY
, 0);
124 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set noise sensitivity";
127 void SetVp9CodecOptions(vpx_codec_ctx_t
* codec
, bool lossless_encode
) {
128 // Request the lowest-CPU usage that VP9 supports, which depends on whether
129 // we are encoding lossy or lossless.
130 // Note that this is configured via the same parameter as for VP8.
131 int cpu_used
= lossless_encode
? 5 : 6;
132 vpx_codec_err_t ret
= vpx_codec_control(codec
, VP8E_SET_CPUUSED
, cpu_used
);
133 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set CPUUSED";
135 // Use the lowest level of noise sensitivity so as to spend less time
136 // on motion estimation and inter-prediction mode.
137 ret
= vpx_codec_control(codec
, VP9E_SET_NOISE_SENSITIVITY
, 0);
138 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set noise sensitivity";
140 // Configure the codec to tune it for screen media.
141 ret
= vpx_codec_control(
142 codec
, VP9E_SET_TUNE_CONTENT
, VP9E_CONTENT_SCREEN
);
143 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set screen content mode";
145 // Set cyclic refresh (aka "top-off") only for lossy encoding.
146 int aq_mode
= lossless_encode
? kVp9AqModeNone
: kVp9AqModeCyclicRefresh
;
147 ret
= vpx_codec_control(codec
, VP9E_SET_AQ_MODE
, aq_mode
);
148 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set aq mode";
151 void FreeImageIfMismatched(bool use_i444
,
152 const webrtc::DesktopSize
& size
,
153 scoped_ptr
<vpx_image_t
>* out_image
,
154 scoped_ptr
<uint8
[]>* out_image_buffer
) {
156 const vpx_img_fmt_t desired_fmt
=
157 use_i444
? VPX_IMG_FMT_I444
: VPX_IMG_FMT_I420
;
158 if (!size
.equals(webrtc::DesktopSize((*out_image
)->w
, (*out_image
)->h
)) ||
159 (*out_image
)->fmt
!= desired_fmt
) {
160 out_image_buffer
->reset();
166 void CreateImage(bool use_i444
,
167 const webrtc::DesktopSize
& size
,
168 scoped_ptr
<vpx_image_t
>* out_image
,
169 scoped_ptr
<uint8
[]>* out_image_buffer
) {
170 DCHECK(!size
.is_empty());
171 DCHECK(!*out_image_buffer
);
174 scoped_ptr
<vpx_image_t
> image(new vpx_image_t());
175 memset(image
.get(), 0, sizeof(vpx_image_t
));
177 // libvpx seems to require both to be assigned.
178 image
->d_w
= size
.width();
179 image
->w
= size
.width();
180 image
->d_h
= size
.height();
181 image
->h
= size
.height();
183 // libvpx should derive chroma shifts from|fmt| but currently has a bug:
184 // https://code.google.com/p/webm/issues/detail?id=627
186 image
->fmt
= VPX_IMG_FMT_I444
;
187 image
->x_chroma_shift
= 0;
188 image
->y_chroma_shift
= 0;
190 image
->fmt
= VPX_IMG_FMT_YV12
;
191 image
->x_chroma_shift
= 1;
192 image
->y_chroma_shift
= 1;
195 // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad
196 // the Y, U and V planes' strides to multiples of 16 bytes.
197 const int y_stride
= ((image
->w
- 1) & ~15) + 16;
198 const int uv_unaligned_stride
= y_stride
>> image
->x_chroma_shift
;
199 const int uv_stride
= ((uv_unaligned_stride
- 1) & ~15) + 16;
201 // libvpx accesses the source image in macro blocks, and will over-read
202 // if the image is not padded out to the next macroblock: crbug.com/119633.
203 // Pad the Y, U and V planes' height out to compensate.
204 // Assuming macroblocks are 16x16, aligning the planes' strides above also
205 // macroblock aligned them.
206 static_assert(kMacroBlockSize
== 16, "macroblock_size_not_16");
207 const int y_rows
= ((image
->h
- 1) & ~(kMacroBlockSize
-1)) + kMacroBlockSize
;
208 const int uv_rows
= y_rows
>> image
->y_chroma_shift
;
210 // Allocate a YUV buffer large enough for the aligned data & padding.
211 const int buffer_size
= y_stride
* y_rows
+ 2*uv_stride
* uv_rows
;
212 scoped_ptr
<uint8
[]> image_buffer(new uint8
[buffer_size
]);
214 // Reset image value to 128 so we just need to fill in the y plane.
215 memset(image_buffer
.get(), 128, buffer_size
);
217 // Fill in the information for |image_|.
218 unsigned char* uchar_buffer
=
219 reinterpret_cast<unsigned char*>(image_buffer
.get());
220 image
->planes
[0] = uchar_buffer
;
221 image
->planes
[1] = image
->planes
[0] + y_stride
* y_rows
;
222 image
->planes
[2] = image
->planes
[1] + uv_stride
* uv_rows
;
223 image
->stride
[0] = y_stride
;
224 image
->stride
[1] = uv_stride
;
225 image
->stride
[2] = uv_stride
;
227 *out_image
= image
.Pass();
228 *out_image_buffer
= image_buffer
.Pass();
234 scoped_ptr
<VideoEncoderVpx
> VideoEncoderVpx::CreateForVP8() {
235 return make_scoped_ptr(new VideoEncoderVpx(false));
239 scoped_ptr
<VideoEncoderVpx
> VideoEncoderVpx::CreateForVP9() {
240 return make_scoped_ptr(new VideoEncoderVpx(true));
243 VideoEncoderVpx::~VideoEncoderVpx() {}
245 void VideoEncoderVpx::SetLosslessEncode(bool want_lossless
) {
246 if (use_vp9_
&& (want_lossless
!= lossless_encode_
)) {
247 lossless_encode_
= want_lossless
;
249 Configure(webrtc::DesktopSize(codec_
->config
.enc
->g_w
,
250 codec_
->config
.enc
->g_h
));
254 void VideoEncoderVpx::SetLosslessColor(bool want_lossless
) {
255 if (use_vp9_
&& (want_lossless
!= lossless_color_
)) {
256 lossless_color_
= want_lossless
;
257 // TODO(wez): Switch to ConfigureCodec() path once libvpx supports it.
258 // See https://code.google.com/p/webm/issues/detail?id=913.
260 // Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
261 // codec_->config.enc->g_h));
266 scoped_ptr
<VideoPacket
> VideoEncoderVpx::Encode(
267 const webrtc::DesktopFrame
& frame
) {
268 DCHECK_LE(32, frame
.size().width());
269 DCHECK_LE(32, frame
.size().height());
271 base::TimeTicks encode_start_time
= base::TimeTicks::Now();
273 // Create or reconfigure the codec to match the size of |frame|.
276 !frame
.size().equals(webrtc::DesktopSize(image_
->w
, image_
->h
)))) {
277 Configure(frame
.size());
280 // Convert the updated capture data ready for encode.
281 webrtc::DesktopRegion updated_region
;
282 PrepareImage(frame
, &updated_region
);
284 // Update active map based on updated region.
285 SetActiveMapFromRegion(updated_region
);
287 // Apply active map to the encoder.
288 vpx_active_map_t act_map
;
289 act_map
.rows
= active_map_height_
;
290 act_map
.cols
= active_map_width_
;
291 act_map
.active_map
= active_map_
.get();
292 if (vpx_codec_control(codec_
.get(), VP8E_SET_ACTIVEMAP
, &act_map
)) {
293 LOG(ERROR
) << "Unable to apply active map";
296 // Do the actual encoding.
297 int timestamp
= (encode_start_time
- timestamp_base_
).InMilliseconds();
298 vpx_codec_err_t ret
= vpx_codec_encode(
299 codec_
.get(), image_
.get(), timestamp
, 1, 0, VPX_DL_REALTIME
);
300 DCHECK_EQ(ret
, VPX_CODEC_OK
)
301 << "Encoding error: " << vpx_codec_err_to_string(ret
) << "\n"
302 << "Details: " << vpx_codec_error(codec_
.get()) << "\n"
303 << vpx_codec_error_detail(codec_
.get());
305 if (use_vp9_
&& !lossless_encode_
) {
306 ret
= vpx_codec_control(codec_
.get(), VP9E_GET_ACTIVEMAP
, &act_map
);
307 DCHECK_EQ(ret
, VPX_CODEC_OK
)
308 << "Failed to fetch active map: "
309 << vpx_codec_err_to_string(ret
) << "\n";
310 UpdateRegionFromActiveMap(&updated_region
);
313 // Read the encoded data.
314 vpx_codec_iter_t iter
= NULL
;
315 bool got_data
= false;
317 // TODO(hclam): Make sure we get exactly one frame from the packet.
318 // TODO(hclam): We should provide the output buffer to avoid one copy.
319 scoped_ptr
<VideoPacket
> packet(
320 helper_
.CreateVideoPacketWithUpdatedRegion(frame
, updated_region
));
321 packet
->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8
);
324 const vpx_codec_cx_pkt_t
* vpx_packet
=
325 vpx_codec_get_cx_data(codec_
.get(), &iter
);
329 switch (vpx_packet
->kind
) {
330 case VPX_CODEC_CX_FRAME_PKT
:
332 packet
->set_data(vpx_packet
->data
.frame
.buf
, vpx_packet
->data
.frame
.sz
);
339 // Note the time taken to encode the pixel data.
340 packet
->set_encode_time_ms(
341 (base::TimeTicks::Now() - encode_start_time
).InMillisecondsRoundedUp());
343 return packet
.Pass();
346 VideoEncoderVpx::VideoEncoderVpx(bool use_vp9
)
348 lossless_encode_(false),
349 lossless_color_(false),
350 active_map_width_(0),
351 active_map_height_(0) {
353 // Use I444 colour space, by default, if specified on the command-line.
354 if (base::CommandLine::ForCurrentProcess()->HasSwitch(
355 kEnableI444SwitchName
)) {
356 SetLosslessColor(true);
361 void VideoEncoderVpx::Configure(const webrtc::DesktopSize
& size
) {
362 DCHECK(use_vp9_
|| !lossless_color_
);
363 DCHECK(use_vp9_
|| !lossless_encode_
);
365 // Tear down |image_| if it no longer matches the size and color settings.
366 // PrepareImage() will then create a new buffer of the required dimensions if
367 // |image_| is not allocated.
368 FreeImageIfMismatched(lossless_color_
, size
, &image_
, &image_buffer_
);
370 // Initialize active map.
371 active_map_width_
= (size
.width() + kMacroBlockSize
- 1) / kMacroBlockSize
;
372 active_map_height_
= (size
.height() + kMacroBlockSize
- 1) / kMacroBlockSize
;
373 active_map_
.reset(new uint8
[active_map_width_
* active_map_height_
]);
375 // TODO(wez): Remove this hack once VPX can handle frame size reconfiguration.
376 // See https://code.google.com/p/webm/issues/detail?id=912.
378 // If the frame size has changed then force re-creation of the codec.
379 if (codec_
->config
.enc
->g_w
!= static_cast<unsigned int>(size
.width()) ||
380 codec_
->config
.enc
->g_h
!= static_cast<unsigned int>(size
.height())) {
385 // (Re)Set the base for frame timestamps if the codec is being (re)created.
387 timestamp_base_
= base::TimeTicks::Now();
390 // Fetch a default configuration for the desired codec.
391 const vpx_codec_iface_t
* interface
=
392 use_vp9_
? vpx_codec_vp9_cx() : vpx_codec_vp8_cx();
393 vpx_codec_enc_cfg_t config
;
394 vpx_codec_err_t ret
= vpx_codec_enc_config_default(interface
, &config
, 0);
395 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to fetch default configuration";
397 // Customize the default configuration to our needs.
399 SetVp9CodecParameters(&config
, size
, lossless_color_
, lossless_encode_
);
401 SetVp8CodecParameters(&config
, size
);
404 // Initialize or re-configure the codec with the custom configuration.
406 codec_
.reset(new vpx_codec_ctx_t
);
407 ret
= vpx_codec_enc_init(codec_
.get(), interface
, &config
, 0);
408 CHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to initialize codec";
410 ret
= vpx_codec_enc_config_set(codec_
.get(), &config
);
411 CHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to reconfigure codec";
414 // Apply further customizations to the codec now it's initialized.
416 SetVp9CodecOptions(codec_
.get(), lossless_encode_
);
418 SetVp8CodecOptions(codec_
.get());
422 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame
& frame
,
423 webrtc::DesktopRegion
* updated_region
) {
424 if (frame
.updated_region().is_empty()) {
425 updated_region
->Clear();
429 updated_region
->Clear();
431 // Pad each rectangle to avoid the block-artefact filters in libvpx from
432 // introducing artefacts; VP9 includes up to 8px either side, and VP8 up to
433 // 3px, so unchanged pixels up to that far out may still be affected by the
434 // changes in the updated region, and so must be listed in the active map.
435 // After padding we align each rectangle to 16x16 active-map macroblocks.
436 // This implicitly ensures all rects have even top-left coords, which is
437 // is required by ConvertRGBToYUVWithRect().
438 // TODO(wez): Do we still need 16x16 align, or is even alignment sufficient?
439 int padding
= use_vp9_
? 8 : 3;
440 for (webrtc::DesktopRegion::Iterator
r(frame
.updated_region());
441 !r
.IsAtEnd(); r
.Advance()) {
442 const webrtc::DesktopRect
& rect
= r
.rect();
443 updated_region
->AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
444 rect
.left() - padding
, rect
.top() - padding
, rect
.right() + padding
,
445 rect
.bottom() + padding
)));
447 DCHECK(!updated_region
->is_empty());
449 // Clip back to the screen dimensions, in case they're not macroblock
450 // aligned. The conversion routines don't require even width & height,
451 // so this is safe even if the source dimensions are not even.
452 updated_region
->IntersectWith(
453 webrtc::DesktopRect::MakeWH(image_
->w
, image_
->h
));
455 CreateImage(lossless_color_
, frame
.size(), &image_
, &image_buffer_
);
456 updated_region
->AddRect(webrtc::DesktopRect::MakeWH(image_
->w
, image_
->h
));
459 // Convert the updated region to YUV ready for encoding.
460 const uint8
* rgb_data
= frame
.data();
461 const int rgb_stride
= frame
.stride();
462 const int y_stride
= image_
->stride
[0];
463 DCHECK_EQ(image_
->stride
[1], image_
->stride
[2]);
464 const int uv_stride
= image_
->stride
[1];
465 uint8
* y_data
= image_
->planes
[0];
466 uint8
* u_data
= image_
->planes
[1];
467 uint8
* v_data
= image_
->planes
[2];
469 switch (image_
->fmt
) {
470 case VPX_IMG_FMT_I444
:
471 for (webrtc::DesktopRegion::Iterator
r(*updated_region
); !r
.IsAtEnd();
473 const webrtc::DesktopRect
& rect
= r
.rect();
474 int rgb_offset
= rgb_stride
* rect
.top() +
475 rect
.left() * kBytesPerRgbPixel
;
476 int yuv_offset
= uv_stride
* rect
.top() + rect
.left();
477 libyuv::ARGBToI444(rgb_data
+ rgb_offset
, rgb_stride
,
478 y_data
+ yuv_offset
, y_stride
,
479 u_data
+ yuv_offset
, uv_stride
,
480 v_data
+ yuv_offset
, uv_stride
,
481 rect
.width(), rect
.height());
484 case VPX_IMG_FMT_YV12
:
485 for (webrtc::DesktopRegion::Iterator
r(*updated_region
); !r
.IsAtEnd();
487 const webrtc::DesktopRect
& rect
= r
.rect();
488 int rgb_offset
= rgb_stride
* rect
.top() +
489 rect
.left() * kBytesPerRgbPixel
;
490 int y_offset
= y_stride
* rect
.top() + rect
.left();
491 int uv_offset
= uv_stride
* rect
.top() / 2 + rect
.left() / 2;
492 libyuv::ARGBToI420(rgb_data
+ rgb_offset
, rgb_stride
,
493 y_data
+ y_offset
, y_stride
,
494 u_data
+ uv_offset
, uv_stride
,
495 v_data
+ uv_offset
, uv_stride
,
496 rect
.width(), rect
.height());
505 void VideoEncoderVpx::SetActiveMapFromRegion(
506 const webrtc::DesktopRegion
& updated_region
) {
507 // Clear active map first.
508 memset(active_map_
.get(), 0, active_map_width_
* active_map_height_
);
510 // Mark updated areas active.
511 for (webrtc::DesktopRegion::Iterator
r(updated_region
); !r
.IsAtEnd();
513 const webrtc::DesktopRect
& rect
= r
.rect();
514 int left
= rect
.left() / kMacroBlockSize
;
515 int right
= (rect
.right() - 1) / kMacroBlockSize
;
516 int top
= rect
.top() / kMacroBlockSize
;
517 int bottom
= (rect
.bottom() - 1) / kMacroBlockSize
;
518 DCHECK_LT(right
, active_map_width_
);
519 DCHECK_LT(bottom
, active_map_height_
);
521 uint8
* map
= active_map_
.get() + top
* active_map_width_
;
522 for (int y
= top
; y
<= bottom
; ++y
) {
523 for (int x
= left
; x
<= right
; ++x
)
525 map
+= active_map_width_
;
530 void VideoEncoderVpx::UpdateRegionFromActiveMap(
531 webrtc::DesktopRegion
* updated_region
) {
532 const uint8
* map
= active_map_
.get();
533 for (int y
= 0; y
< active_map_height_
; ++y
) {
534 for (int x0
= 0; x0
< active_map_width_
;) {
536 for (; x1
< active_map_width_
; ++x1
) {
537 if (map
[y
* active_map_width_
+ x1
] == 0)
541 updated_region
->AddRect(webrtc::DesktopRect::MakeLTRB(
542 kMacroBlockSize
* x0
, kMacroBlockSize
* y
, kMacroBlockSize
* x1
,
543 kMacroBlockSize
* (y
+ 1)));
548 updated_region
->IntersectWith(
549 webrtc::DesktopRect::MakeWH(image_
->w
, image_
->h
));
552 } // namespace remoting