remoting/codec/video_encoder_vpx.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/codec/video_encoder_vpx.h"
   6
   7 #include "base/bind.h"
   8 #include "base/command_line.h"
   9 #include "base/logging.h"
  10 #include "base/sys_info.h"
  11 #include "remoting/base/util.h"
  12 #include "remoting/proto/video.pb.h"
  13 #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
  14 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  15 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
  16 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
  17
  18 extern "C" {
  19 #define VPX_CODEC_DISABLE_COMPAT 1
  20 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
  21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  22 }
  23
  24 namespace remoting {
  25
  26 namespace {
  27
  28 // Name of command-line flag to enable VP9 to use I444 by default.
  29 const char kEnableI444SwitchName[] = "enable-i444";
  30
  31 // Number of bytes in an RGBx pixel.
  32 const int kBytesPerRgbPixel = 4;
  33
  34 // Defines the dimension of a macro block. This is used to compute the active
  35 // map for the encoder.
  36 const int kMacroBlockSize = 16;
  37
  38 // Magic encoder profile numbers for I420 and I444 input formats.
  39 const int kVp9I420ProfileNumber = 0;
  40 const int kVp9I444ProfileNumber = 1;
  41
  42 void SetCommonCodecParameters(vpx_codec_enc_cfg_t* config,
  43                               const webrtc::DesktopSize& size) {
  44   // Use millisecond granularity time base.
  45   config->g_timebase.num = 1;
  46   config->g_timebase.den = 1000;
  47
  48   config->g_w = size.width();
  49   config->g_h = size.height();
  50   config->g_pass = VPX_RC_ONE_PASS;
  51
  52   // Start emitting packets immediately.
  53   config->g_lag_in_frames = 0;
  54
  55   // Since the transport layer is reliable, keyframes should not be necessary.
  56   // However, due to crbug.com/440223, decoding fails after 30,000 non-key
  57   // frames, so take the hit of an "unnecessary" key-frame every 10,000 frames.
  58   config->kf_min_dist = 10000;
  59   config->kf_max_dist = 10000;
  60
  61   // Using 2 threads gives a great boost in performance for most systems with
  62   // adequate processing power. NB: Going to multiple threads on low end
  63   // windows systems can really hurt performance.
  64   // http://crbug.com/99179
  65   config->g_threads = (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1;
  66 }
  67
  68 void SetVp8CodecParameters(vpx_codec_enc_cfg_t* config,
  69                            const webrtc::DesktopSize& size) {
  70   // Adjust default target bit-rate to account for actual desktop size.
  71   config->rc_target_bitrate = size.width() * size.height() *
  72       config->rc_target_bitrate / config->g_w / config->g_h;
  73
  74   SetCommonCodecParameters(config, size);
  75
  76   // Value of 2 means using the real time profile. This is basically a
  77   // redundant option since we explicitly select real time mode when doing
  78   // encoding.
  79   config->g_profile = 2;
  80
  81   // Clamping the quantizer constrains the worst-case quality and CPU usage.
  82   config->rc_min_quantizer = 20;
  83   config->rc_max_quantizer = 30;
  84 }
  85
  86 void SetVp9CodecParameters(vpx_codec_enc_cfg_t* config,
  87                            const webrtc::DesktopSize& size,
  88                            bool lossless_color,
  89                            bool lossless_encode) {
  90   SetCommonCodecParameters(config, size);
  91
  92   // Configure VP9 for I420 or I444 source frames.
  93   config->g_profile =
  94       lossless_color ? kVp9I444ProfileNumber : kVp9I420ProfileNumber;
  95
  96   if (lossless_encode) {
  97     // Disable quantization entirely, putting the encoder in "lossless" mode.
  98     config->rc_min_quantizer = 0;
  99     config->rc_max_quantizer = 0;
 100     config->rc_end_usage = VPX_VBR;
 101   } else {
 102     config->rc_min_quantizer = 4;
 103     config->rc_max_quantizer = 30;
 104     config->rc_end_usage = VPX_CBR;
 105     // In the absence of a good bandwidth estimator set the target bitrate to a
 106     // conservative default.
 107     config->rc_target_bitrate = 500;
 108   }
 109 }
 110
 111 void SetVp8CodecOptions(vpx_codec_ctx_t* codec) {
 112   // CPUUSED of 16 will have the smallest CPU load. This turns off sub-pixel
 113   // motion search.
 114   vpx_codec_err_t ret = vpx_codec_control(codec, VP8E_SET_CPUUSED, 16);
 115   DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set CPUUSED";
 116
 117   // Use the lowest level of noise sensitivity so as to spend less time
 118   // on motion estimation and inter-prediction mode.
 119   ret = vpx_codec_control(codec, VP8E_SET_NOISE_SENSITIVITY, 0);
 120   DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set noise sensitivity";
 121 }
 122
 123 void SetVp9CodecOptions(vpx_codec_ctx_t* codec, bool lossless_encode) {
 124   // Request the lowest-CPU usage that VP9 supports, which depends on whether
 125   // we are encoding lossy or lossless.
 126   // Note that this is configured via the same parameter as for VP8.
 127   int cpu_used = lossless_encode ? 5 : 6;
 128   vpx_codec_err_t ret = vpx_codec_control(codec, VP8E_SET_CPUUSED, cpu_used);
 129   DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set CPUUSED";
 130
 131   // Use the lowest level of noise sensitivity so as to spend less time
 132   // on motion estimation and inter-prediction mode.
 133   ret = vpx_codec_control(codec, VP9E_SET_NOISE_SENSITIVITY, 0);
 134   DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set noise sensitivity";
 135
 136   // Configure the codec to tune it for screen media.
 137   ret = vpx_codec_control(
 138       codec, VP9E_SET_TUNE_CONTENT, VP9E_CONTENT_SCREEN);
 139   DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set screen content mode";
 140 }
 141
 142 void FreeImageIfMismatched(bool use_i444,
 143                            const webrtc::DesktopSize& size,
 144                            scoped_ptr<vpx_image_t>* out_image,
 145                            scoped_ptr<uint8[]>* out_image_buffer) {
 146   if (*out_image) {
 147     const vpx_img_fmt_t desired_fmt =
 148         use_i444 ? VPX_IMG_FMT_I444 : VPX_IMG_FMT_I420;
 149     if (!size.equals(webrtc::DesktopSize((*out_image)->w, (*out_image)->h)) ||
 150         (*out_image)->fmt != desired_fmt) {
 151       out_image_buffer->reset();
 152       out_image->reset();
 153     }
 154   }
 155 }
 156
 157 void CreateImage(bool use_i444,
 158                  const webrtc::DesktopSize& size,
 159                  scoped_ptr<vpx_image_t>* out_image,
 160                  scoped_ptr<uint8[]>* out_image_buffer) {
 161   DCHECK(!size.is_empty());
 162   DCHECK(!*out_image_buffer);
 163   DCHECK(!*out_image);
 164
 165   scoped_ptr<vpx_image_t> image(new vpx_image_t());
 166   memset(image.get(), 0, sizeof(vpx_image_t));
 167
 168   // libvpx seems to require both to be assigned.
 169   image->d_w = size.width();
 170   image->w = size.width();
 171   image->d_h = size.height();
 172   image->h = size.height();
 173
 174   // libvpx should derive chroma shifts from|fmt| but currently has a bug:
 175   // https://code.google.com/p/webm/issues/detail?id=627
 176   if (use_i444) {
 177     image->fmt = VPX_IMG_FMT_I444;
 178     image->x_chroma_shift = 0;
 179     image->y_chroma_shift = 0;
 180   } else { // I420
 181     image->fmt = VPX_IMG_FMT_YV12;
 182     image->x_chroma_shift = 1;
 183     image->y_chroma_shift = 1;
 184   }
 185
 186   // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad
 187   // the Y, U and V planes' strides to multiples of 16 bytes.
 188   const int y_stride = ((image->w - 1) & ~15) + 16;
 189   const int uv_unaligned_stride = y_stride >> image->x_chroma_shift;
 190   const int uv_stride = ((uv_unaligned_stride - 1) & ~15) + 16;
 191
 192   // libvpx accesses the source image in macro blocks, and will over-read
 193   // if the image is not padded out to the next macroblock: crbug.com/119633.
 194   // Pad the Y, U and V planes' height out to compensate.
 195   // Assuming macroblocks are 16x16, aligning the planes' strides above also
 196   // macroblock aligned them.
 197   static_assert(kMacroBlockSize == 16, "macroblock_size_not_16");
 198   const int y_rows = ((image->h - 1) & ~(kMacroBlockSize-1)) + kMacroBlockSize;
 199   const int uv_rows = y_rows >> image->y_chroma_shift;
 200
 201   // Allocate a YUV buffer large enough for the aligned data & padding.
 202   const int buffer_size = y_stride * y_rows + 2*uv_stride * uv_rows;
 203   scoped_ptr<uint8[]> image_buffer(new uint8[buffer_size]);
 204
 205   // Reset image value to 128 so we just need to fill in the y plane.
 206   memset(image_buffer.get(), 128, buffer_size);
 207
 208   // Fill in the information for |image_|.
 209   unsigned char* uchar_buffer =
 210       reinterpret_cast<unsigned char*>(image_buffer.get());
 211   image->planes[0] = uchar_buffer;
 212   image->planes[1] = image->planes[0] + y_stride * y_rows;
 213   image->planes[2] = image->planes[1] + uv_stride * uv_rows;
 214   image->stride[0] = y_stride;
 215   image->stride[1] = uv_stride;
 216   image->stride[2] = uv_stride;
 217
 218   *out_image = image.Pass();
 219   *out_image_buffer = image_buffer.Pass();
 220 }
 221
 222 } // namespace
 223
 224 // static
 225 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP8() {
 226   return make_scoped_ptr(new VideoEncoderVpx(false));
 227 }
 228
 229 // static
 230 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP9() {
 231   return make_scoped_ptr(new VideoEncoderVpx(true));
 232 }
 233
 234 VideoEncoderVpx::~VideoEncoderVpx() {}
 235
 236 void VideoEncoderVpx::SetLosslessEncode(bool want_lossless) {
 237   if (use_vp9_ && (want_lossless != lossless_encode_)) {
 238     lossless_encode_ = want_lossless;
 239     if (codec_)
 240       Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
 241                                     codec_->config.enc->g_h));
 242   }
 243 }
 244
 245 void VideoEncoderVpx::SetLosslessColor(bool want_lossless) {
 246   if (use_vp9_ && (want_lossless != lossless_color_)) {
 247     lossless_color_ = want_lossless;
 248     // TODO(wez): Switch to ConfigureCodec() path once libvpx supports it.
 249     // See https://code.google.com/p/webm/issues/detail?id=913.
 250     //if (codec_)
 251     //  Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
 252     //                                codec_->config.enc->g_h));
 253     codec_.reset();
 254   }
 255 }
 256
 257 scoped_ptr<VideoPacket> VideoEncoderVpx::Encode(
 258     const webrtc::DesktopFrame& frame) {
 259   DCHECK_LE(32, frame.size().width());
 260   DCHECK_LE(32, frame.size().height());
 261
 262   base::TimeTicks encode_start_time = base::TimeTicks::Now();
 263
 264   // Create or reconfigure the codec to match the size of |frame|.
 265   if (!codec_ ||
 266       (image_ &&
 267        !frame.size().equals(webrtc::DesktopSize(image_->w, image_->h)))) {
 268     Configure(frame.size());
 269   }
 270
 271   // Convert the updated capture data ready for encode.
 272   webrtc::DesktopRegion updated_region;
 273   PrepareImage(frame, &updated_region);
 274
 275   // Update active map based on updated region.
 276   PrepareActiveMap(updated_region);
 277
 278   // Apply active map to the encoder.
 279   vpx_active_map_t act_map;
 280   act_map.rows = active_map_height_;
 281   act_map.cols = active_map_width_;
 282   act_map.active_map = active_map_.get();
 283   if (vpx_codec_control(codec_.get(), VP8E_SET_ACTIVEMAP, &act_map)) {
 284     LOG(ERROR) << "Unable to apply active map";
 285   }
 286
 287   // Do the actual encoding.
 288   int timestamp = (encode_start_time - timestamp_base_).InMilliseconds();
 289   vpx_codec_err_t ret = vpx_codec_encode(
 290       codec_.get(), image_.get(), timestamp, 1, 0, VPX_DL_REALTIME);
 291   DCHECK_EQ(ret, VPX_CODEC_OK)
 292       << "Encoding error: " << vpx_codec_err_to_string(ret) << "\n"
 293       << "Details: " << vpx_codec_error(codec_.get()) << "\n"
 294       << vpx_codec_error_detail(codec_.get());
 295
 296   // Read the encoded data.
 297   vpx_codec_iter_t iter = NULL;
 298   bool got_data = false;
 299
 300   // TODO(hclam): Make sure we get exactly one frame from the packet.
 301   // TODO(hclam): We should provide the output buffer to avoid one copy.
 302   scoped_ptr<VideoPacket> packet(
 303       helper_.CreateVideoPacketWithUpdatedRegion(frame, updated_region));
 304   packet->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8);
 305
 306   while (!got_data) {
 307     const vpx_codec_cx_pkt_t* vpx_packet =
 308         vpx_codec_get_cx_data(codec_.get(), &iter);
 309     if (!vpx_packet)
 310       continue;
 311
 312     switch (vpx_packet->kind) {
 313       case VPX_CODEC_CX_FRAME_PKT:
 314         got_data = true;
 315         packet->set_data(vpx_packet->data.frame.buf, vpx_packet->data.frame.sz);
 316         break;
 317       default:
 318         break;
 319     }
 320   }
 321
 322   // Note the time taken to encode the pixel data.
 323   packet->set_encode_time_ms(
 324       (base::TimeTicks::Now() - encode_start_time).InMillisecondsRoundedUp());
 325
 326   return packet.Pass();
 327 }
 328
 329 VideoEncoderVpx::VideoEncoderVpx(bool use_vp9)
 330     : use_vp9_(use_vp9),
 331       lossless_encode_(false),
 332       lossless_color_(false),
 333       active_map_width_(0),
 334       active_map_height_(0) {
 335   if (use_vp9_) {
 336     // Use I444 colour space, by default, if specified on the command-line.
 337     if (base::CommandLine::ForCurrentProcess()->HasSwitch(
 338             kEnableI444SwitchName)) {
 339       SetLosslessColor(true);
 340     }
 341   }
 342 }
 343
 344 void VideoEncoderVpx::Configure(const webrtc::DesktopSize& size) {
 345   DCHECK(use_vp9_ || !lossless_color_);
 346   DCHECK(use_vp9_ || !lossless_encode_);
 347
 348   // Tear down |image_| if it no longer matches the size and color settings.
 349   // PrepareImage() will then create a new buffer of the required dimensions if
 350   // |image_| is not allocated.
 351   FreeImageIfMismatched(lossless_color_, size, &image_, &image_buffer_);
 352
 353   // Initialize active map.
 354   active_map_width_ = (size.width() + kMacroBlockSize - 1) / kMacroBlockSize;
 355   active_map_height_ = (size.height() + kMacroBlockSize - 1) / kMacroBlockSize;
 356   active_map_.reset(new uint8[active_map_width_ * active_map_height_]);
 357
 358   // TODO(wez): Remove this hack once VPX can handle frame size reconfiguration.
 359   // See https://code.google.com/p/webm/issues/detail?id=912.
 360   if (codec_) {
 361     // If the frame size has changed then force re-creation of the codec.
 362     if (codec_->config.enc->g_w != static_cast<unsigned int>(size.width()) ||
 363         codec_->config.enc->g_h != static_cast<unsigned int>(size.height())) {
 364       codec_.reset();
 365     }
 366   }
 367
 368   // (Re)Set the base for frame timestamps if the codec is being (re)created.
 369   if (!codec_) {
 370     timestamp_base_ = base::TimeTicks::Now();
 371   }
 372
 373   // Fetch a default configuration for the desired codec.
 374   const vpx_codec_iface_t* interface =
 375       use_vp9_ ? vpx_codec_vp9_cx() : vpx_codec_vp8_cx();
 376   vpx_codec_enc_cfg_t config;
 377   vpx_codec_err_t ret = vpx_codec_enc_config_default(interface, &config, 0);
 378   DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to fetch default configuration";
 379
 380   // Customize the default configuration to our needs.
 381   if (use_vp9_) {
 382     SetVp9CodecParameters(&config, size, lossless_color_, lossless_encode_);
 383   } else {
 384     SetVp8CodecParameters(&config, size);
 385   }
 386
 387   // Initialize or re-configure the codec with the custom configuration.
 388   if (!codec_) {
 389     codec_.reset(new vpx_codec_ctx_t);
 390     ret = vpx_codec_enc_init(codec_.get(), interface, &config, 0);
 391     CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to initialize codec";
 392   } else {
 393     ret = vpx_codec_enc_config_set(codec_.get(), &config);
 394     CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to reconfigure codec";
 395   }
 396
 397   // Apply further customizations to the codec now it's initialized.
 398   if (use_vp9_) {
 399     SetVp9CodecOptions(codec_.get(), lossless_encode_);
 400   } else {
 401     SetVp8CodecOptions(codec_.get());
 402   }
 403 }
 404
 405 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame& frame,
 406                                    webrtc::DesktopRegion* updated_region) {
 407   if (frame.updated_region().is_empty()) {
 408     updated_region->Clear();
 409     return;
 410   }
 411
 412   updated_region->Clear();
 413   if (image_) {
 414     // Pad each rectangle to avoid the block-artefact filters in libvpx from
 415     // introducing artefacts; VP9 includes up to 8px either side, and VP8 up to
 416     // 3px, so unchanged pixels up to that far out may still be affected by the
 417     // changes in the updated region, and so must be listed in the active map.
 418     // After padding we align each rectangle to 16x16 active-map macroblocks.
 419     // This implicitly ensures all rects have even top-left coords, which is
 420     // is required by ConvertRGBToYUVWithRect().
 421     // TODO(wez): Do we still need 16x16 align, or is even alignment sufficient?
 422     int padding = use_vp9_ ? 8 : 3;
 423     for (webrtc::DesktopRegion::Iterator r(frame.updated_region());
 424          !r.IsAtEnd(); r.Advance()) {
 425       const webrtc::DesktopRect& rect = r.rect();
 426       updated_region->AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
 427           rect.left() - padding, rect.top() - padding, rect.right() + padding,
 428           rect.bottom() + padding)));
 429     }
 430     DCHECK(!updated_region->is_empty());
 431
 432     // Clip back to the screen dimensions, in case they're not macroblock
 433     // aligned. The conversion routines don't require even width & height,
 434     // so this is safe even if the source dimensions are not even.
 435     updated_region->IntersectWith(
 436         webrtc::DesktopRect::MakeWH(image_->w, image_->h));
 437   } else {
 438     CreateImage(lossless_color_, frame.size(), &image_, &image_buffer_);
 439     updated_region->AddRect(webrtc::DesktopRect::MakeWH(image_->w, image_->h));
 440   }
 441
 442   // Convert the updated region to YUV ready for encoding.
 443   const uint8* rgb_data = frame.data();
 444   const int rgb_stride = frame.stride();
 445   const int y_stride = image_->stride[0];
 446   DCHECK_EQ(image_->stride[1], image_->stride[2]);
 447   const int uv_stride = image_->stride[1];
 448   uint8* y_data = image_->planes[0];
 449   uint8* u_data = image_->planes[1];
 450   uint8* v_data = image_->planes[2];
 451
 452   switch (image_->fmt) {
 453     case VPX_IMG_FMT_I444:
 454       for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
 455            r.Advance()) {
 456         const webrtc::DesktopRect& rect = r.rect();
 457         int rgb_offset = rgb_stride * rect.top() +
 458                          rect.left() * kBytesPerRgbPixel;
 459         int yuv_offset = uv_stride * rect.top() + rect.left();
 460         libyuv::ARGBToI444(rgb_data + rgb_offset, rgb_stride,
 461                            y_data + yuv_offset, y_stride,
 462                            u_data + yuv_offset, uv_stride,
 463                            v_data + yuv_offset, uv_stride,
 464                            rect.width(), rect.height());
 465       }
 466       break;
 467     case VPX_IMG_FMT_YV12:
 468       for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
 469            r.Advance()) {
 470         const webrtc::DesktopRect& rect = r.rect();
 471         int rgb_offset = rgb_stride * rect.top() +
 472                          rect.left() * kBytesPerRgbPixel;
 473         int y_offset = y_stride * rect.top() + rect.left();
 474         int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2;
 475         libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride,
 476                            y_data + y_offset, y_stride,
 477                            u_data + uv_offset, uv_stride,
 478                            v_data + uv_offset, uv_stride,
 479                            rect.width(), rect.height());
 480       }
 481       break;
 482     default:
 483       NOTREACHED();
 484       break;
 485   }
 486 }
 487
 488 void VideoEncoderVpx::PrepareActiveMap(
 489     const webrtc::DesktopRegion& updated_region) {
 490   // Clear active map first.
 491   memset(active_map_.get(), 0, active_map_width_ * active_map_height_);
 492
 493   // Mark updated areas active.
 494   for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
 495        r.Advance()) {
 496     const webrtc::DesktopRect& rect = r.rect();
 497     int left = rect.left() / kMacroBlockSize;
 498     int right = (rect.right() - 1) / kMacroBlockSize;
 499     int top = rect.top() / kMacroBlockSize;
 500     int bottom = (rect.bottom() - 1) / kMacroBlockSize;
 501     DCHECK_LT(right, active_map_width_);
 502     DCHECK_LT(bottom, active_map_height_);
 503
 504     uint8* map = active_map_.get() + top * active_map_width_;
 505     for (int y = top; y <= bottom; ++y) {
 506       for (int x = left; x <= right; ++x)
 507         map[x] = 1;
 508       map += active_map_width_;
 509     }
 510   }
 511 }
 512
 513 }  // namespace remoting