1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "remoting/codec/video_encoder_vpx.h"
8 #include "base/logging.h"
9 #include "base/sys_info.h"
10 #include "remoting/base/util.h"
11 #include "remoting/proto/video.pb.h"
12 #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
13 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
14 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
15 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
18 #define VPX_CODEC_DISABLE_COMPAT 1
19 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
20 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
27 // Number of bytes in an RGBx pixel.
28 const int kBytesPerRgbPixel
= 4;
30 // Defines the dimension of a macro block. This is used to compute the active
31 // map for the encoder.
32 const int kMacroBlockSize
= 16;
34 // Magic encoder profile numbers for I420 and I444 input formats.
35 const int kVp9I420ProfileNumber
= 0;
36 const int kVp9I444ProfileNumber
= 1;
38 // Magic encoder constants for adaptive quantization strategy.
39 const int kVp9AqModeNone
= 0;
40 const int kVp9AqModeCyclicRefresh
= 3;
42 void SetCommonCodecParameters(vpx_codec_enc_cfg_t
* config
,
43 const webrtc::DesktopSize
& size
) {
44 // Use millisecond granularity time base.
45 config
->g_timebase
.num
= 1;
46 config
->g_timebase
.den
= 1000;
48 config
->g_w
= size
.width();
49 config
->g_h
= size
.height();
50 config
->g_pass
= VPX_RC_ONE_PASS
;
52 // Start emitting packets immediately.
53 config
->g_lag_in_frames
= 0;
55 // Since the transport layer is reliable, keyframes should not be necessary.
56 // However, due to crbug.com/440223, decoding fails after 30,000 non-key
57 // frames, so take the hit of an "unnecessary" key-frame every 10,000 frames.
58 config
->kf_min_dist
= 10000;
59 config
->kf_max_dist
= 10000;
61 // Using 2 threads gives a great boost in performance for most systems with
62 // adequate processing power. NB: Going to multiple threads on low end
63 // windows systems can really hurt performance.
64 // http://crbug.com/99179
65 config
->g_threads
= (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1;
68 void SetVp8CodecParameters(vpx_codec_enc_cfg_t
* config
,
69 const webrtc::DesktopSize
& size
) {
70 // Adjust default target bit-rate to account for actual desktop size.
71 config
->rc_target_bitrate
= size
.width() * size
.height() *
72 config
->rc_target_bitrate
/ config
->g_w
/ config
->g_h
;
74 SetCommonCodecParameters(config
, size
);
76 // Value of 2 means using the real time profile. This is basically a
77 // redundant option since we explicitly select real time mode when doing
79 config
->g_profile
= 2;
81 // Clamping the quantizer constrains the worst-case quality and CPU usage.
82 config
->rc_min_quantizer
= 20;
83 config
->rc_max_quantizer
= 30;
86 void SetVp9CodecParameters(vpx_codec_enc_cfg_t
* config
,
87 const webrtc::DesktopSize
& size
,
89 bool lossless_encode
) {
90 SetCommonCodecParameters(config
, size
);
92 // Configure VP9 for I420 or I444 source frames.
94 lossless_color
? kVp9I444ProfileNumber
: kVp9I420ProfileNumber
;
96 if (lossless_encode
) {
97 // Disable quantization entirely, putting the encoder in "lossless" mode.
98 config
->rc_min_quantizer
= 0;
99 config
->rc_max_quantizer
= 0;
100 config
->rc_end_usage
= VPX_VBR
;
102 // TODO(wez): Set quantization range to 4-40, once the libvpx encoder is
103 // updated not to output any bits if nothing needs topping-off.
104 config
->rc_min_quantizer
= 20;
105 config
->rc_max_quantizer
= 30;
106 config
->rc_end_usage
= VPX_CBR
;
107 // In the absence of a good bandwidth estimator set the target bitrate to a
108 // conservative default.
109 config
->rc_target_bitrate
= 500;
113 void SetVp8CodecOptions(vpx_codec_ctx_t
* codec
) {
114 // CPUUSED of 16 will have the smallest CPU load. This turns off sub-pixel
116 vpx_codec_err_t ret
= vpx_codec_control(codec
, VP8E_SET_CPUUSED
, 16);
117 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set CPUUSED";
119 // Use the lowest level of noise sensitivity so as to spend less time
120 // on motion estimation and inter-prediction mode.
121 ret
= vpx_codec_control(codec
, VP8E_SET_NOISE_SENSITIVITY
, 0);
122 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set noise sensitivity";
125 void SetVp9CodecOptions(vpx_codec_ctx_t
* codec
, bool lossless_encode
) {
126 // Request the lowest-CPU usage that VP9 supports, which depends on whether
127 // we are encoding lossy or lossless.
128 // Note that this is configured via the same parameter as for VP8.
129 int cpu_used
= lossless_encode
? 5 : 6;
130 vpx_codec_err_t ret
= vpx_codec_control(codec
, VP8E_SET_CPUUSED
, cpu_used
);
131 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set CPUUSED";
133 // Use the lowest level of noise sensitivity so as to spend less time
134 // on motion estimation and inter-prediction mode.
135 ret
= vpx_codec_control(codec
, VP9E_SET_NOISE_SENSITIVITY
, 0);
136 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set noise sensitivity";
138 // Configure the codec to tune it for screen media.
139 ret
= vpx_codec_control(
140 codec
, VP9E_SET_TUNE_CONTENT
, VP9E_CONTENT_SCREEN
);
141 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set screen content mode";
143 // Set cyclic refresh (aka "top-off") only for lossy encoding.
144 int aq_mode
= lossless_encode
? kVp9AqModeNone
: kVp9AqModeCyclicRefresh
;
145 ret
= vpx_codec_control(codec
, VP9E_SET_AQ_MODE
, aq_mode
);
146 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to set aq mode";
149 void FreeImageIfMismatched(bool use_i444
,
150 const webrtc::DesktopSize
& size
,
151 scoped_ptr
<vpx_image_t
>* out_image
,
152 scoped_ptr
<uint8
[]>* out_image_buffer
) {
154 const vpx_img_fmt_t desired_fmt
=
155 use_i444
? VPX_IMG_FMT_I444
: VPX_IMG_FMT_I420
;
156 if (!size
.equals(webrtc::DesktopSize((*out_image
)->w
, (*out_image
)->h
)) ||
157 (*out_image
)->fmt
!= desired_fmt
) {
158 out_image_buffer
->reset();
164 void CreateImage(bool use_i444
,
165 const webrtc::DesktopSize
& size
,
166 scoped_ptr
<vpx_image_t
>* out_image
,
167 scoped_ptr
<uint8
[]>* out_image_buffer
) {
168 DCHECK(!size
.is_empty());
169 DCHECK(!*out_image_buffer
);
172 scoped_ptr
<vpx_image_t
> image(new vpx_image_t());
173 memset(image
.get(), 0, sizeof(vpx_image_t
));
175 // libvpx seems to require both to be assigned.
176 image
->d_w
= size
.width();
177 image
->w
= size
.width();
178 image
->d_h
= size
.height();
179 image
->h
= size
.height();
181 // libvpx should derive chroma shifts from|fmt| but currently has a bug:
182 // https://code.google.com/p/webm/issues/detail?id=627
184 image
->fmt
= VPX_IMG_FMT_I444
;
185 image
->x_chroma_shift
= 0;
186 image
->y_chroma_shift
= 0;
188 image
->fmt
= VPX_IMG_FMT_YV12
;
189 image
->x_chroma_shift
= 1;
190 image
->y_chroma_shift
= 1;
193 // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad
194 // the Y, U and V planes' strides to multiples of 16 bytes.
195 const int y_stride
= ((image
->w
- 1) & ~15) + 16;
196 const int uv_unaligned_stride
= y_stride
>> image
->x_chroma_shift
;
197 const int uv_stride
= ((uv_unaligned_stride
- 1) & ~15) + 16;
199 // libvpx accesses the source image in macro blocks, and will over-read
200 // if the image is not padded out to the next macroblock: crbug.com/119633.
201 // Pad the Y, U and V planes' height out to compensate.
202 // Assuming macroblocks are 16x16, aligning the planes' strides above also
203 // macroblock aligned them.
204 static_assert(kMacroBlockSize
== 16, "macroblock_size_not_16");
205 const int y_rows
= ((image
->h
- 1) & ~(kMacroBlockSize
-1)) + kMacroBlockSize
;
206 const int uv_rows
= y_rows
>> image
->y_chroma_shift
;
208 // Allocate a YUV buffer large enough for the aligned data & padding.
209 const int buffer_size
= y_stride
* y_rows
+ 2*uv_stride
* uv_rows
;
210 scoped_ptr
<uint8
[]> image_buffer(new uint8
[buffer_size
]);
212 // Reset image value to 128 so we just need to fill in the y plane.
213 memset(image_buffer
.get(), 128, buffer_size
);
215 // Fill in the information for |image_|.
216 unsigned char* uchar_buffer
=
217 reinterpret_cast<unsigned char*>(image_buffer
.get());
218 image
->planes
[0] = uchar_buffer
;
219 image
->planes
[1] = image
->planes
[0] + y_stride
* y_rows
;
220 image
->planes
[2] = image
->planes
[1] + uv_stride
* uv_rows
;
221 image
->stride
[0] = y_stride
;
222 image
->stride
[1] = uv_stride
;
223 image
->stride
[2] = uv_stride
;
225 *out_image
= image
.Pass();
226 *out_image_buffer
= image_buffer
.Pass();
232 scoped_ptr
<VideoEncoderVpx
> VideoEncoderVpx::CreateForVP8() {
233 return make_scoped_ptr(new VideoEncoderVpx(false));
237 scoped_ptr
<VideoEncoderVpx
> VideoEncoderVpx::CreateForVP9() {
238 return make_scoped_ptr(new VideoEncoderVpx(true));
241 VideoEncoderVpx::~VideoEncoderVpx() {}
243 void VideoEncoderVpx::SetLosslessEncode(bool want_lossless
) {
244 if (use_vp9_
&& (want_lossless
!= lossless_encode_
)) {
245 lossless_encode_
= want_lossless
;
247 Configure(webrtc::DesktopSize(codec_
->config
.enc
->g_w
,
248 codec_
->config
.enc
->g_h
));
252 void VideoEncoderVpx::SetLosslessColor(bool want_lossless
) {
253 if (use_vp9_
&& (want_lossless
!= lossless_color_
)) {
254 lossless_color_
= want_lossless
;
255 // TODO(wez): Switch to ConfigureCodec() path once libvpx supports it.
256 // See https://code.google.com/p/webm/issues/detail?id=913.
258 // Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
259 // codec_->config.enc->g_h));
264 scoped_ptr
<VideoPacket
> VideoEncoderVpx::Encode(
265 const webrtc::DesktopFrame
& frame
) {
266 DCHECK_LE(32, frame
.size().width());
267 DCHECK_LE(32, frame
.size().height());
269 // If there is nothing to encode, and nothing to top-off, then return nothing.
270 if (frame
.updated_region().is_empty() && !encode_unchanged_frame_
)
273 base::TimeTicks encode_start_time
= base::TimeTicks::Now();
275 // Create or reconfigure the codec to match the size of |frame|.
278 !frame
.size().equals(webrtc::DesktopSize(image_
->w
, image_
->h
)))) {
279 Configure(frame
.size());
282 // Convert the updated capture data ready for encode.
283 webrtc::DesktopRegion updated_region
;
284 PrepareImage(frame
, &updated_region
);
286 // Update active map based on updated region.
287 SetActiveMapFromRegion(updated_region
);
289 // Apply active map to the encoder.
290 vpx_active_map_t act_map
;
291 act_map
.rows
= active_map_size_
.height();
292 act_map
.cols
= active_map_size_
.width();
293 act_map
.active_map
= active_map_
.get();
294 if (vpx_codec_control(codec_
.get(), VP8E_SET_ACTIVEMAP
, &act_map
)) {
295 LOG(ERROR
) << "Unable to apply active map";
298 // Do the actual encoding.
299 int timestamp
= (encode_start_time
- timestamp_base_
).InMilliseconds();
300 vpx_codec_err_t ret
= vpx_codec_encode(
301 codec_
.get(), image_
.get(), timestamp
, 1, 0, VPX_DL_REALTIME
);
302 DCHECK_EQ(ret
, VPX_CODEC_OK
)
303 << "Encoding error: " << vpx_codec_err_to_string(ret
) << "\n"
304 << "Details: " << vpx_codec_error(codec_
.get()) << "\n"
305 << vpx_codec_error_detail(codec_
.get());
307 if (use_vp9_
&& !lossless_encode_
) {
308 ret
= vpx_codec_control(codec_
.get(), VP9E_GET_ACTIVEMAP
, &act_map
);
309 DCHECK_EQ(ret
, VPX_CODEC_OK
)
310 << "Failed to fetch active map: "
311 << vpx_codec_err_to_string(ret
) << "\n";
312 UpdateRegionFromActiveMap(&updated_region
);
314 // If the encoder output no changes then there's nothing left to top-off.
315 encode_unchanged_frame_
= !updated_region
.is_empty();
318 // Read the encoded data.
319 vpx_codec_iter_t iter
= nullptr;
320 bool got_data
= false;
322 // TODO(hclam): Make sure we get exactly one frame from the packet.
323 // TODO(hclam): We should provide the output buffer to avoid one copy.
324 scoped_ptr
<VideoPacket
> packet(
325 helper_
.CreateVideoPacketWithUpdatedRegion(frame
, updated_region
));
326 packet
->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8
);
329 const vpx_codec_cx_pkt_t
* vpx_packet
=
330 vpx_codec_get_cx_data(codec_
.get(), &iter
);
334 switch (vpx_packet
->kind
) {
335 case VPX_CODEC_CX_FRAME_PKT
:
337 packet
->set_data(vpx_packet
->data
.frame
.buf
, vpx_packet
->data
.frame
.sz
);
344 // Note the time taken to encode the pixel data.
345 packet
->set_encode_time_ms(
346 (base::TimeTicks::Now() - encode_start_time
).InMillisecondsRoundedUp());
348 return packet
.Pass();
351 VideoEncoderVpx::VideoEncoderVpx(bool use_vp9
)
352 : use_vp9_(use_vp9
), encode_unchanged_frame_(false) {
355 void VideoEncoderVpx::Configure(const webrtc::DesktopSize
& size
) {
356 DCHECK(use_vp9_
|| !lossless_color_
);
357 DCHECK(use_vp9_
|| !lossless_encode_
);
359 // Tear down |image_| if it no longer matches the size and color settings.
360 // PrepareImage() will then create a new buffer of the required dimensions if
361 // |image_| is not allocated.
362 FreeImageIfMismatched(lossless_color_
, size
, &image_
, &image_buffer_
);
364 // Initialize active map.
365 active_map_size_
= webrtc::DesktopSize(
366 (size
.width() + kMacroBlockSize
- 1) / kMacroBlockSize
,
367 (size
.height() + kMacroBlockSize
- 1) / kMacroBlockSize
);
369 new uint8
[active_map_size_
.width() * active_map_size_
.height()]);
371 // TODO(wez): Remove this hack once VPX can handle frame size reconfiguration.
372 // See https://code.google.com/p/webm/issues/detail?id=912.
374 // If the frame size has changed then force re-creation of the codec.
375 if (codec_
->config
.enc
->g_w
!= static_cast<unsigned int>(size
.width()) ||
376 codec_
->config
.enc
->g_h
!= static_cast<unsigned int>(size
.height())) {
381 // (Re)Set the base for frame timestamps if the codec is being (re)created.
383 timestamp_base_
= base::TimeTicks::Now();
386 // Fetch a default configuration for the desired codec.
387 const vpx_codec_iface_t
* interface
=
388 use_vp9_
? vpx_codec_vp9_cx() : vpx_codec_vp8_cx();
389 vpx_codec_enc_cfg_t config
;
390 vpx_codec_err_t ret
= vpx_codec_enc_config_default(interface
, &config
, 0);
391 DCHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to fetch default configuration";
393 // Customize the default configuration to our needs.
395 SetVp9CodecParameters(&config
, size
, lossless_color_
, lossless_encode_
);
397 SetVp8CodecParameters(&config
, size
);
400 // Initialize or re-configure the codec with the custom configuration.
402 codec_
.reset(new vpx_codec_ctx_t
);
403 ret
= vpx_codec_enc_init(codec_
.get(), interface
, &config
, 0);
404 CHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to initialize codec";
406 ret
= vpx_codec_enc_config_set(codec_
.get(), &config
);
407 CHECK_EQ(VPX_CODEC_OK
, ret
) << "Failed to reconfigure codec";
410 // Apply further customizations to the codec now it's initialized.
412 SetVp9CodecOptions(codec_
.get(), lossless_encode_
);
414 SetVp8CodecOptions(codec_
.get());
418 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame
& frame
,
419 webrtc::DesktopRegion
* updated_region
) {
420 if (frame
.updated_region().is_empty()) {
421 updated_region
->Clear();
425 updated_region
->Clear();
427 // Pad each rectangle to avoid the block-artefact filters in libvpx from
428 // introducing artefacts; VP9 includes up to 8px either side, and VP8 up to
429 // 3px, so unchanged pixels up to that far out may still be affected by the
430 // changes in the updated region, and so must be listed in the active map.
431 // After padding we align each rectangle to 16x16 active-map macroblocks.
432 // This implicitly ensures all rects have even top-left coords, which is
433 // is required by ConvertRGBToYUVWithRect().
434 // TODO(wez): Do we still need 16x16 align, or is even alignment sufficient?
435 int padding
= use_vp9_
? 8 : 3;
436 for (webrtc::DesktopRegion::Iterator
r(frame
.updated_region());
437 !r
.IsAtEnd(); r
.Advance()) {
438 const webrtc::DesktopRect
& rect
= r
.rect();
439 updated_region
->AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
440 rect
.left() - padding
, rect
.top() - padding
, rect
.right() + padding
,
441 rect
.bottom() + padding
)));
443 DCHECK(!updated_region
->is_empty());
445 // Clip back to the screen dimensions, in case they're not macroblock
446 // aligned. The conversion routines don't require even width & height,
447 // so this is safe even if the source dimensions are not even.
448 updated_region
->IntersectWith(
449 webrtc::DesktopRect::MakeWH(image_
->w
, image_
->h
));
451 CreateImage(lossless_color_
, frame
.size(), &image_
, &image_buffer_
);
452 updated_region
->AddRect(webrtc::DesktopRect::MakeWH(image_
->w
, image_
->h
));
455 // Convert the updated region to YUV ready for encoding.
456 const uint8
* rgb_data
= frame
.data();
457 const int rgb_stride
= frame
.stride();
458 const int y_stride
= image_
->stride
[0];
459 DCHECK_EQ(image_
->stride
[1], image_
->stride
[2]);
460 const int uv_stride
= image_
->stride
[1];
461 uint8
* y_data
= image_
->planes
[0];
462 uint8
* u_data
= image_
->planes
[1];
463 uint8
* v_data
= image_
->planes
[2];
465 switch (image_
->fmt
) {
466 case VPX_IMG_FMT_I444
:
467 for (webrtc::DesktopRegion::Iterator
r(*updated_region
); !r
.IsAtEnd();
469 const webrtc::DesktopRect
& rect
= r
.rect();
470 int rgb_offset
= rgb_stride
* rect
.top() +
471 rect
.left() * kBytesPerRgbPixel
;
472 int yuv_offset
= uv_stride
* rect
.top() + rect
.left();
473 libyuv::ARGBToI444(rgb_data
+ rgb_offset
, rgb_stride
,
474 y_data
+ yuv_offset
, y_stride
,
475 u_data
+ yuv_offset
, uv_stride
,
476 v_data
+ yuv_offset
, uv_stride
,
477 rect
.width(), rect
.height());
480 case VPX_IMG_FMT_YV12
:
481 for (webrtc::DesktopRegion::Iterator
r(*updated_region
); !r
.IsAtEnd();
483 const webrtc::DesktopRect
& rect
= r
.rect();
484 int rgb_offset
= rgb_stride
* rect
.top() +
485 rect
.left() * kBytesPerRgbPixel
;
486 int y_offset
= y_stride
* rect
.top() + rect
.left();
487 int uv_offset
= uv_stride
* rect
.top() / 2 + rect
.left() / 2;
488 libyuv::ARGBToI420(rgb_data
+ rgb_offset
, rgb_stride
,
489 y_data
+ y_offset
, y_stride
,
490 u_data
+ uv_offset
, uv_stride
,
491 v_data
+ uv_offset
, uv_stride
,
492 rect
.width(), rect
.height());
501 void VideoEncoderVpx::SetActiveMapFromRegion(
502 const webrtc::DesktopRegion
& updated_region
) {
503 // Clear active map first.
504 memset(active_map_
.get(), 0,
505 active_map_size_
.width() * active_map_size_
.height());
507 // Mark updated areas active.
508 for (webrtc::DesktopRegion::Iterator
r(updated_region
); !r
.IsAtEnd();
510 const webrtc::DesktopRect
& rect
= r
.rect();
511 int left
= rect
.left() / kMacroBlockSize
;
512 int right
= (rect
.right() - 1) / kMacroBlockSize
;
513 int top
= rect
.top() / kMacroBlockSize
;
514 int bottom
= (rect
.bottom() - 1) / kMacroBlockSize
;
515 DCHECK_LT(right
, active_map_size_
.width());
516 DCHECK_LT(bottom
, active_map_size_
.height());
518 uint8
* map
= active_map_
.get() + top
* active_map_size_
.width();
519 for (int y
= top
; y
<= bottom
; ++y
) {
520 for (int x
= left
; x
<= right
; ++x
)
522 map
+= active_map_size_
.width();
527 void VideoEncoderVpx::UpdateRegionFromActiveMap(
528 webrtc::DesktopRegion
* updated_region
) {
529 const uint8
* map
= active_map_
.get();
530 for (int y
= 0; y
< active_map_size_
.height(); ++y
) {
531 for (int x0
= 0; x0
< active_map_size_
.width();) {
533 for (; x1
< active_map_size_
.width(); ++x1
) {
534 if (map
[y
* active_map_size_
.width() + x1
] == 0)
538 updated_region
->AddRect(webrtc::DesktopRect::MakeLTRB(
539 kMacroBlockSize
* x0
, kMacroBlockSize
* y
, kMacroBlockSize
* x1
,
540 kMacroBlockSize
* (y
+ 1)));
545 updated_region
->IntersectWith(
546 webrtc::DesktopRect::MakeWH(image_
->w
, image_
->h
));
549 } // namespace remoting