remoting/codec/video_encoder_vpx.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "remoting/codec/video_encoder_vpx.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/sys_info.h"
  10 #include "base/time/time.h"
  11 #include "media/base/yuv_convert.h"
  12 #include "remoting/base/util.h"
  13 #include "remoting/proto/video.pb.h"
  14 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  15 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
  16 #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
  17
  18 extern "C" {
  19 #define VPX_CODEC_DISABLE_COMPAT 1
  20 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
  21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  22 }
  23
  24 namespace remoting {
  25
  26 namespace {
  27
  28 // Defines the dimension of a macro block. This is used to compute the active
  29 // map for the encoder.
  30 const int kMacroBlockSize = 16;
  31
  32 ScopedVpxCodec CreateVP8Codec(const webrtc::DesktopSize& size) {
  33   ScopedVpxCodec codec(new vpx_codec_ctx_t);
  34
  35   // Configure the encoder.
  36   vpx_codec_enc_cfg_t config;
  37   const vpx_codec_iface_t* algo = vpx_codec_vp8_cx();
  38   CHECK(algo);
  39   vpx_codec_err_t ret = vpx_codec_enc_config_default(algo, &config, 0);
  40   if (ret != VPX_CODEC_OK)
  41     return ScopedVpxCodec();
  42
  43   config.rc_target_bitrate = size.width() * size.height() *
  44       config.rc_target_bitrate / config.g_w / config.g_h;
  45   config.g_w = size.width();
  46   config.g_h = size.height();
  47   config.g_pass = VPX_RC_ONE_PASS;
  48
  49   // Value of 2 means using the real time profile. This is basically a
  50   // redundant option since we explicitly select real time mode when doing
  51   // encoding.
  52   config.g_profile = 2;
  53
  54   // Using 2 threads gives a great boost in performance for most systems with
  55   // adequate processing power. NB: Going to multiple threads on low end
  56   // windows systems can really hurt performance.
  57   // http://crbug.com/99179
  58   config.g_threads = (base::SysInfo::NumberOfProcessors() > 2) ? 2 : 1;
  59   config.rc_min_quantizer = 20;
  60   config.rc_max_quantizer = 30;
  61   config.g_timebase.num = 1;
  62   config.g_timebase.den = 20;
  63
  64   if (vpx_codec_enc_init(codec.get(), algo, &config, 0))
  65     return ScopedVpxCodec();
  66
  67   // Value of 16 will have the smallest CPU load. This turns off subpixel
  68   // motion search.
  69   if (vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, 16))
  70     return ScopedVpxCodec();
  71
  72   // Use the lowest level of noise sensitivity so as to spend less time
  73   // on motion estimation and inter-prediction mode.
  74   if (vpx_codec_control(codec.get(), VP8E_SET_NOISE_SENSITIVITY, 0))
  75     return ScopedVpxCodec();
  76
  77   return codec.Pass();
  78 }
  79
  80 ScopedVpxCodec CreateVP9Codec(const webrtc::DesktopSize& size) {
  81   ScopedVpxCodec codec(new vpx_codec_ctx_t);
  82
  83   // Configure the encoder.
  84   vpx_codec_enc_cfg_t config;
  85   const vpx_codec_iface_t* algo = vpx_codec_vp9_cx();
  86   CHECK(algo);
  87   vpx_codec_err_t ret = vpx_codec_enc_config_default(algo, &config, 0);
  88   if (ret != VPX_CODEC_OK)
  89     return ScopedVpxCodec();
  90
  91   //config.rc_target_bitrate = size.width() * size.height() *
  92   //    config.rc_target_bitrate / config.g_w / config.g_h;
  93   config.g_w = size.width();
  94   config.g_h = size.height();
  95   config.g_pass = VPX_RC_ONE_PASS;
  96
  97   // Only the default profile is currently supported for VP9 encoding.
  98   config.g_profile = 0;
  99
 100   // Start emitting packets immediately.
 101   config.g_lag_in_frames = 0;
 102
 103   // Prevent VP9 from ruining output quality with quantization.
 104   config.rc_max_quantizer = 0;
 105
 106   if (vpx_codec_enc_init(codec.get(), algo, &config, 0))
 107     return ScopedVpxCodec();
 108
 109   // VP9 encode doesn't yet support Realtime, so falls back to Good quality,
 110   // for which 4 is the lowest CPU usage.
 111   // Note that this is configured via the same parameter as for VP8.
 112   if (vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, 4))
 113     return ScopedVpxCodec();
 114
 115   // Use the lowest level of noise sensitivity so as to spend less time
 116   // on motion estimation and inter-prediction mode.
 117   // Note that this is configured via the same parameter as for VP8.
 118   if (vpx_codec_control(codec.get(), VP8E_SET_NOISE_SENSITIVITY, 0))
 119     return ScopedVpxCodec();
 120
 121   return codec.Pass();
 122 }
 123
 124 }  // namespace
 125
 126 // static
 127 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP8() {
 128   return scoped_ptr<VideoEncoderVpx>(
 129       new VideoEncoderVpx(base::Bind(&CreateVP8Codec)));
 130 }
 131
 132 // static
 133 scoped_ptr<VideoEncoderVpx> VideoEncoderVpx::CreateForVP9() {
 134   return scoped_ptr<VideoEncoderVpx>(
 135       new VideoEncoderVpx(base::Bind(&CreateVP9Codec)));
 136 }
 137
 138 VideoEncoderVpx::~VideoEncoderVpx() {}
 139
 140 scoped_ptr<VideoPacket> VideoEncoderVpx::Encode(
 141     const webrtc::DesktopFrame& frame) {
 142   DCHECK_LE(32, frame.size().width());
 143   DCHECK_LE(32, frame.size().height());
 144
 145   base::Time encode_start_time = base::Time::Now();
 146
 147   if (!codec_ ||
 148       !frame.size().equals(webrtc::DesktopSize(image_->w, image_->h))) {
 149     bool ret = Initialize(frame.size());
 150     // TODO(hclam): Handle error better.
 151     CHECK(ret) << "Initialization of encoder failed";
 152   }
 153
 154   // Convert the updated capture data ready for encode.
 155   webrtc::DesktopRegion updated_region;
 156   PrepareImage(frame, &updated_region);
 157
 158   // Update active map based on updated region.
 159   PrepareActiveMap(updated_region);
 160
 161   // Apply active map to the encoder.
 162   vpx_active_map_t act_map;
 163   act_map.rows = active_map_height_;
 164   act_map.cols = active_map_width_;
 165   act_map.active_map = active_map_.get();
 166   if (vpx_codec_control(codec_.get(), VP8E_SET_ACTIVEMAP, &act_map)) {
 167     LOG(ERROR) << "Unable to apply active map";
 168   }
 169
 170   // Do the actual encoding.
 171   vpx_codec_err_t ret = vpx_codec_encode(codec_.get(), image_.get(),
 172                                          last_timestamp_,
 173                                          1, 0, VPX_DL_REALTIME);
 174   DCHECK_EQ(ret, VPX_CODEC_OK)
 175       << "Encoding error: " << vpx_codec_err_to_string(ret) << "\n"
 176       << "Details: " << vpx_codec_error(codec_.get()) << "\n"
 177       << vpx_codec_error_detail(codec_.get());
 178
 179   // TODO(hclam): Apply the proper timestamp here.
 180   last_timestamp_ += 50;
 181
 182   // Read the encoded data.
 183   vpx_codec_iter_t iter = NULL;
 184   bool got_data = false;
 185
 186   // TODO(hclam): Make sure we get exactly one frame from the packet.
 187   // TODO(hclam): We should provide the output buffer to avoid one copy.
 188   scoped_ptr<VideoPacket> packet(new VideoPacket());
 189
 190   while (!got_data) {
 191     const vpx_codec_cx_pkt_t* vpx_packet =
 192         vpx_codec_get_cx_data(codec_.get(), &iter);
 193     if (!vpx_packet)
 194       continue;
 195
 196     switch (vpx_packet->kind) {
 197       case VPX_CODEC_CX_FRAME_PKT:
 198         got_data = true;
 199         packet->set_data(vpx_packet->data.frame.buf, vpx_packet->data.frame.sz);
 200         break;
 201       default:
 202         break;
 203     }
 204   }
 205
 206   // Construct the VideoPacket message.
 207   packet->mutable_format()->set_encoding(VideoPacketFormat::ENCODING_VP8);
 208   packet->mutable_format()->set_screen_width(frame.size().width());
 209   packet->mutable_format()->set_screen_height(frame.size().height());
 210   packet->set_capture_time_ms(frame.capture_time_ms());
 211   packet->set_encode_time_ms(
 212       (base::Time::Now() - encode_start_time).InMillisecondsRoundedUp());
 213   if (!frame.dpi().is_zero()) {
 214     packet->mutable_format()->set_x_dpi(frame.dpi().x());
 215     packet->mutable_format()->set_y_dpi(frame.dpi().y());
 216   }
 217   for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
 218        r.Advance()) {
 219     Rect* rect = packet->add_dirty_rects();
 220     rect->set_x(r.rect().left());
 221     rect->set_y(r.rect().top());
 222     rect->set_width(r.rect().width());
 223     rect->set_height(r.rect().height());
 224   }
 225
 226   return packet.Pass();
 227 }
 228
 229 VideoEncoderVpx::VideoEncoderVpx(const InitializeCodecCallback& init_codec)
 230     : init_codec_(init_codec),
 231       active_map_width_(0),
 232       active_map_height_(0),
 233       last_timestamp_(0) {
 234 }
 235
 236 bool VideoEncoderVpx::Initialize(const webrtc::DesktopSize& size) {
 237   codec_.reset();
 238
 239   image_.reset(new vpx_image_t());
 240   memset(image_.get(), 0, sizeof(vpx_image_t));
 241
 242   image_->fmt = VPX_IMG_FMT_YV12;
 243
 244   // libvpx seems to require both to be assigned.
 245   image_->d_w = size.width();
 246   image_->w = size.width();
 247   image_->d_h = size.height();
 248   image_->h = size.height();
 249
 250   // libvpx should derive this from|fmt| but currently has a bug:
 251   // https://code.google.com/p/webm/issues/detail?id=627
 252   image_->x_chroma_shift = 1;
 253   image_->y_chroma_shift = 1;
 254
 255   // Initialize active map.
 256   active_map_width_ = (image_->w + kMacroBlockSize - 1) / kMacroBlockSize;
 257   active_map_height_ = (image_->h + kMacroBlockSize - 1) / kMacroBlockSize;
 258   active_map_.reset(new uint8[active_map_width_ * active_map_height_]);
 259
 260   // libyuv's fast-path requires 16-byte aligned pointers and strides, so pad
 261   // the Y, U and V planes' strides to multiples of 16 bytes.
 262   const int y_stride = ((image_->w - 1) & ~15) + 16;
 263   const int uv_unaligned_stride = y_stride / 2;
 264   const int uv_stride = ((uv_unaligned_stride - 1) & ~15) + 16;
 265
 266   // libvpx accesses the source image in macro blocks, and will over-read
 267   // if the image is not padded out to the next macroblock: crbug.com/119633.
 268   // Pad the Y, U and V planes' height out to compensate.
 269   // Assuming macroblocks are 16x16, aligning the planes' strides above also
 270   // macroblock aligned them.
 271   DCHECK_EQ(16, kMacroBlockSize);
 272   const int y_rows = active_map_height_ * kMacroBlockSize;
 273   const int uv_rows = y_rows / 2;
 274
 275   // Allocate a YUV buffer large enough for the aligned data & padding.
 276   const int buffer_size = y_stride * y_rows + 2 * uv_stride * uv_rows;
 277   yuv_image_.reset(new uint8[buffer_size]);
 278
 279   // Reset image value to 128 so we just need to fill in the y plane.
 280   memset(yuv_image_.get(), 128, buffer_size);
 281
 282   // Fill in the information for |image_|.
 283   unsigned char* image = reinterpret_cast<unsigned char*>(yuv_image_.get());
 284   image_->planes[0] = image;
 285   image_->planes[1] = image_->planes[0] + y_stride * y_rows;
 286   image_->planes[2] = image_->planes[1] + uv_stride * uv_rows;
 287   image_->stride[0] = y_stride;
 288   image_->stride[1] = uv_stride;
 289   image_->stride[2] = uv_stride;
 290
 291   // Initialize the codec.
 292   codec_ = init_codec_.Run(size);
 293
 294   return codec_;
 295 }
 296
 297 void VideoEncoderVpx::PrepareImage(const webrtc::DesktopFrame& frame,
 298                                    webrtc::DesktopRegion* updated_region) {
 299   if (frame.updated_region().is_empty()) {
 300     updated_region->Clear();
 301     return;
 302   }
 303
 304   // Align the region to macroblocks, to avoid encoding artefacts.
 305   // This also ensures that all rectangles have even-aligned top-left, which
 306   // is required for ConvertRGBToYUVWithRect() to work.
 307   std::vector<webrtc::DesktopRect> aligned_rects;
 308   for (webrtc::DesktopRegion::Iterator r(frame.updated_region());
 309        !r.IsAtEnd(); r.Advance()) {
 310     const webrtc::DesktopRect& rect = r.rect();
 311     aligned_rects.push_back(AlignRect(webrtc::DesktopRect::MakeLTRB(
 312         rect.left(), rect.top(), rect.right(), rect.bottom())));
 313   }
 314   DCHECK(!aligned_rects.empty());
 315   updated_region->Clear();
 316   updated_region->AddRects(&aligned_rects[0], aligned_rects.size());
 317
 318   // Clip back to the screen dimensions, in case they're not macroblock aligned.
 319   // The conversion routines don't require even width & height, so this is safe
 320   // even if the source dimensions are not even.
 321   updated_region->IntersectWith(
 322       webrtc::DesktopRect::MakeWH(image_->w, image_->h));
 323
 324   // Convert the updated region to YUV ready for encoding.
 325   const uint8* rgb_data = frame.data();
 326   const int rgb_stride = frame.stride();
 327   const int y_stride = image_->stride[0];
 328   DCHECK_EQ(image_->stride[1], image_->stride[2]);
 329   const int uv_stride = image_->stride[1];
 330   uint8* y_data = image_->planes[0];
 331   uint8* u_data = image_->planes[1];
 332   uint8* v_data = image_->planes[2];
 333   for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
 334        r.Advance()) {
 335     const webrtc::DesktopRect& rect = r.rect();
 336     ConvertRGB32ToYUVWithRect(
 337         rgb_data, y_data, u_data, v_data,
 338         rect.left(), rect.top(), rect.width(), rect.height(),
 339         rgb_stride, y_stride, uv_stride);
 340   }
 341 }
 342
 343 void VideoEncoderVpx::PrepareActiveMap(
 344     const webrtc::DesktopRegion& updated_region) {
 345   // Clear active map first.
 346   memset(active_map_.get(), 0, active_map_width_ * active_map_height_);
 347
 348   // Mark updated areas active.
 349   for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
 350        r.Advance()) {
 351     const webrtc::DesktopRect& rect = r.rect();
 352     int left = rect.left() / kMacroBlockSize;
 353     int right = (rect.right() - 1) / kMacroBlockSize;
 354     int top = rect.top() / kMacroBlockSize;
 355     int bottom = (rect.bottom() - 1) / kMacroBlockSize;
 356     DCHECK_LT(right, active_map_width_);
 357     DCHECK_LT(bottom, active_map_height_);
 358
 359     uint8* map = active_map_.get() + top * active_map_width_;
 360     for (int y = top; y <= bottom; ++y) {
 361       for (int x = left; x <= right; ++x)
 362         map[x] = 1;
 363       map += active_map_width_;
 364     }
 365   }
 366 }
 367
 368 }  // namespace remoting