media/cast/sender/vp8_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/vp8_encoder.h"
   6
   7 #include "base/logging.h"
   8 #include "media/base/video_frame.h"
   9 #include "media/cast/cast_defines.h"
  10 #include "media/cast/net/cast_transport_config.h"
  11 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  12
  13 namespace media {
  14 namespace cast {
  15
  16 namespace {
  17
  18 // After a pause in the video stream, what is the maximum duration amount to
  19 // pass to the encoder for the next frame (in terms of 1/max_fps sized periods)?
  20 // This essentially controls the encoded size of the first frame that follows a
  21 // pause in the video stream.
  22 const int kRestartFramePeriods = 3;
  23
  24 }  // namespace
  25
  26 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config)
  27     : cast_config_(video_config),
  28       use_multiple_video_buffers_(
  29           cast_config_.max_number_of_video_buffers_used ==
  30           kNumberOfVp8VideoBuffers),
  31       key_frame_requested_(true),
  32       bitrate_kbit_(cast_config_.start_bitrate / 1000),
  33       last_encoded_frame_id_(kStartFrameId),
  34       last_acked_frame_id_(kStartFrameId),
  35       undroppable_frames_(0) {
  36   config_.g_timebase.den = 0;  // Not initialized.
  37
  38   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
  39     buffer_state_[i].frame_id = last_encoded_frame_id_;
  40     buffer_state_[i].state = kBufferStartState;
  41   }
  42
  43   // VP8 have 3 buffers available for prediction, with
  44   // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
  45   // however in this mode we can not skip frames in the receiver to catch up
  46   // after a temporary network outage; with max_number_of_video_buffers_used
  47   // set to 3 we allow 2 frames to be skipped by the receiver without error
  48   // propagation.
  49   DCHECK(cast_config_.max_number_of_video_buffers_used == 1 ||
  50          cast_config_.max_number_of_video_buffers_used ==
  51              kNumberOfVp8VideoBuffers)
  52       << "Invalid argument";
  53
  54   thread_checker_.DetachFromThread();
  55 }
  56
  57 Vp8Encoder::~Vp8Encoder() {
  58   DCHECK(thread_checker_.CalledOnValidThread());
  59   if (is_initialized())
  60     vpx_codec_destroy(&encoder_);
  61 }
  62
  63 void Vp8Encoder::Initialize() {
  64   DCHECK(thread_checker_.CalledOnValidThread());
  65   DCHECK(!is_initialized());
  66   // The encoder will be created/configured when the first frame encode is
  67   // requested.
  68 }
  69
  70 void Vp8Encoder::ConfigureForNewFrameSize(const gfx::Size& frame_size) {
  71   if (is_initialized()) {
  72     // Workaround for VP8 bug: If the new size is strictly less-than-or-equal to
  73     // the old size, in terms of area, the existing encoder instance can
  74     // continue.  Otherwise, completely tear-down and re-create a new encoder to
  75     // avoid a shutdown crash.
  76     if (frame_size.GetArea() <= gfx::Size(config_.g_w, config_.g_h).GetArea() &&
  77         !use_multiple_video_buffers_) {
  78       DVLOG(1) << "Continuing to use existing encoder at smaller frame size: "
  79                << gfx::Size(config_.g_w, config_.g_h).ToString() << " --> "
  80                << frame_size.ToString();
  81       config_.g_w = frame_size.width();
  82       config_.g_h = frame_size.height();
  83       if (vpx_codec_enc_config_set(&encoder_, &config_) == VPX_CODEC_OK)
  84         return;
  85       DVLOG(1) << "libvpx rejected the attempt to use a smaller frame size in "
  86                   "the current instance.";
  87     }
  88
  89     DVLOG(1) << "Destroying/Re-Creating encoder for larger frame size: "
  90              << gfx::Size(config_.g_w, config_.g_h).ToString() << " --> "
  91              << frame_size.ToString();
  92     vpx_codec_destroy(&encoder_);
  93   } else {
  94     DVLOG(1) << "Creating encoder for the first frame; size: "
  95              << frame_size.ToString();
  96   }
  97
  98   // Reset multi-buffer mode state.
  99   last_acked_frame_id_ = last_encoded_frame_id_;
 100   undroppable_frames_ = 0;
 101   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 102     buffer_state_[i].frame_id = last_encoded_frame_id_;
 103     buffer_state_[i].state = kBufferStartState;
 104   }
 105
 106   // Populate encoder configuration with default values.
 107   CHECK_EQ(vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config_, 0),
 108            VPX_CODEC_OK);
 109
 110   config_.g_threads = cast_config_.number_of_encode_threads;
 111   config_.g_w = frame_size.width();
 112   config_.g_h = frame_size.height();
 113   // Set the timebase to match that of base::TimeDelta.
 114   config_.g_timebase.num = 1;
 115   config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
 116   if (use_multiple_video_buffers_) {
 117     // We must enable error resilience when we use multiple buffers, due to
 118     // codec requirements.
 119     config_.g_error_resilient = 1;
 120   }
 121   // |g_pass| and |g_lag_in_frames| must be "one pass" and zero, respectively,
 122   // in order for VP8 to support changing frame sizes during encoding:
 123   config_.g_pass = VPX_RC_ONE_PASS;
 124   config_.g_lag_in_frames = 0;  // Immediate data output for each frame.
 125
 126   // Rate control settings.
 127   config_.rc_dropframe_thresh = 0;  // The encoder may not drop any frames.
 128   config_.rc_resize_allowed = 0;  // TODO(miu): Why not?  Investigate this.
 129   config_.rc_end_usage = VPX_CBR;
 130   config_.rc_target_bitrate = bitrate_kbit_;
 131   config_.rc_min_quantizer = cast_config_.min_qp;
 132   config_.rc_max_quantizer = cast_config_.max_qp;
 133   // TODO(miu): Revisit these now that the encoder is being successfully
 134   // micro-managed.
 135   config_.rc_undershoot_pct = 100;
 136   config_.rc_overshoot_pct = 15;
 137   // TODO(miu): Document why these rc_buf_*_sz values were chosen and/or
 138   // research for better values.  Should they be computed from the target
 139   // playout delay?
 140   config_.rc_buf_initial_sz = 500;
 141   config_.rc_buf_optimal_sz = 600;
 142   config_.rc_buf_sz = 1000;
 143
 144   config_.kf_mode = VPX_KF_DISABLED;
 145
 146   vpx_codec_flags_t flags = 0;
 147   CHECK_EQ(vpx_codec_enc_init(&encoder_, vpx_codec_vp8_cx(), &config_, flags),
 148            VPX_CODEC_OK);
 149
 150   // Raise the threshold for considering macroblocks as static.  The default is
 151   // zero, so this setting makes the encoder less sensitive to motion.  This
 152   // lowers the probability of needing to utilize more CPU to search for motion
 153   // vectors.
 154   CHECK_EQ(vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1),
 155            VPX_CODEC_OK);
 156
 157   // Improve quality by enabling sets of codec features that utilize more CPU.
 158   // The default is zero, with increasingly more CPU to be used as the value is
 159   // more negative.
 160   // TODO(miu): Document why this value was chosen and expected behaviors.
 161   // Should this be dynamic w.r.t. hardware performance?
 162   CHECK_EQ(vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -6), VPX_CODEC_OK);
 163 }
 164
 165 void Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame,
 166                         const base::TimeTicks& reference_time,
 167                         EncodedFrame* encoded_frame) {
 168   DCHECK(thread_checker_.CalledOnValidThread());
 169   DCHECK(encoded_frame);
 170
 171   // Initialize on-demand.  Later, if the video frame size has changed, update
 172   // the encoder configuration.
 173   const gfx::Size frame_size = video_frame->visible_rect().size();
 174   if (!is_initialized() || gfx::Size(config_.g_w, config_.g_h) != frame_size)
 175     ConfigureForNewFrameSize(frame_size);
 176
 177   uint32 latest_frame_id_to_reference;
 178   Vp8Buffers buffer_to_update;
 179   vpx_codec_flags_t flags = 0;
 180   if (key_frame_requested_) {
 181     flags = VPX_EFLAG_FORCE_KF;
 182     // Self reference.
 183     latest_frame_id_to_reference = last_encoded_frame_id_ + 1;
 184     // We can pick any buffer as buffer_to_update since we update
 185     // them all.
 186     buffer_to_update = kLastBuffer;
 187   } else {
 188     // Reference all acked frames (buffers).
 189     latest_frame_id_to_reference = GetCodecReferenceFlags(&flags);
 190     buffer_to_update = GetNextBufferToUpdate();
 191     GetCodecUpdateFlags(buffer_to_update, &flags);
 192   }
 193
 194   // Wrapper for vpx_codec_encode() to access the YUV data in the |video_frame|.
 195   // Only the VISIBLE rectangle within |video_frame| is exposed to the codec.
 196   vpx_image_t vpx_image;
 197   vpx_image_t* const result = vpx_img_wrap(
 198       &vpx_image,
 199       VPX_IMG_FMT_I420,
 200       frame_size.width(),
 201       frame_size.height(),
 202       1,
 203       video_frame->data(VideoFrame::kYPlane));
 204   DCHECK_EQ(result, &vpx_image);
 205   vpx_image.planes[VPX_PLANE_Y] =
 206       video_frame->visible_data(VideoFrame::kYPlane);
 207   vpx_image.planes[VPX_PLANE_U] =
 208       video_frame->visible_data(VideoFrame::kUPlane);
 209   vpx_image.planes[VPX_PLANE_V] =
 210       video_frame->visible_data(VideoFrame::kVPlane);
 211   vpx_image.stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane);
 212   vpx_image.stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane);
 213   vpx_image.stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane);
 214
 215   // The frame duration given to the VP8 codec affects a number of important
 216   // behaviors, including: per-frame bandwidth, CPU time spent encoding,
 217   // temporal quality trade-offs, and key/golden/alt-ref frame generation
 218   // intervals.  Use the actual amount of time between the current and previous
 219   // frames as a prediction for the next frame's duration, but bound the
 220   // prediction to account for the fact that the frame rate can be highly
 221   // variable, including long pauses in the video stream.
 222   const base::TimeDelta minimum_frame_duration =
 223       base::TimeDelta::FromSecondsD(1.0 / cast_config_.max_frame_rate);
 224   const base::TimeDelta maximum_frame_duration =
 225       base::TimeDelta::FromSecondsD(static_cast<double>(kRestartFramePeriods) /
 226                                         cast_config_.max_frame_rate);
 227   const base::TimeDelta last_frame_duration =
 228       video_frame->timestamp() - last_frame_timestamp_;
 229   const base::TimeDelta predicted_frame_duration =
 230       std::max(minimum_frame_duration,
 231                std::min(maximum_frame_duration, last_frame_duration));
 232   last_frame_timestamp_ = video_frame->timestamp();
 233
 234   // Encode the frame.  The presentation time stamp argument here is fixed to
 235   // zero to force the encoder to base its single-frame bandwidth calculations
 236   // entirely on |predicted_frame_duration| and the target bitrate setting being
 237   // micro-managed via calls to UpdateRates().
 238   CHECK_EQ(vpx_codec_encode(&encoder_,
 239                             &vpx_image,
 240                             0,
 241                             predicted_frame_duration.InMicroseconds(),
 242                             flags,
 243                             VPX_DL_REALTIME),
 244            VPX_CODEC_OK)
 245       << "BUG: Invalid arguments passed to vpx_codec_encode().";
 246
 247   // Pull data from the encoder, populating a new EncodedFrame.
 248   encoded_frame->frame_id = ++last_encoded_frame_id_;
 249   const vpx_codec_cx_pkt_t* pkt = NULL;
 250   vpx_codec_iter_t iter = NULL;
 251   while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) {
 252     if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
 253       continue;
 254     if (pkt->data.frame.flags & VPX_FRAME_IS_KEY) {
 255       // TODO(hubbe): Replace "dependency" with a "bool is_key_frame".
 256       encoded_frame->dependency = EncodedFrame::KEY;
 257       encoded_frame->referenced_frame_id = encoded_frame->frame_id;
 258     } else {
 259       encoded_frame->dependency = EncodedFrame::DEPENDENT;
 260       // Frame dependencies could theoretically be relaxed by looking for the
 261       // VPX_FRAME_IS_DROPPABLE flag, but in recent testing (Oct 2014), this
 262       // flag never seems to be set.
 263       encoded_frame->referenced_frame_id = latest_frame_id_to_reference;
 264     }
 265     encoded_frame->rtp_timestamp =
 266         TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency);
 267     encoded_frame->reference_time = reference_time;
 268     encoded_frame->data.assign(
 269         static_cast<const uint8*>(pkt->data.frame.buf),
 270         static_cast<const uint8*>(pkt->data.frame.buf) + pkt->data.frame.sz);
 271     break;  // Done, since all data is provided in one CX_FRAME_PKT packet.
 272   }
 273   DCHECK(!encoded_frame->data.empty())
 274       << "BUG: Encoder must provide data since lagged encoding is disabled.";
 275
 276   DVLOG(2) << "VP8 encoded frame_id " << encoded_frame->frame_id
 277            << ", sized:" << encoded_frame->data.size();
 278
 279   if (encoded_frame->dependency == EncodedFrame::KEY) {
 280     key_frame_requested_ = false;
 281
 282     for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 283       buffer_state_[i].state = kBufferSent;
 284       buffer_state_[i].frame_id = encoded_frame->frame_id;
 285     }
 286   } else {
 287     if (buffer_to_update != kNoBuffer) {
 288       buffer_state_[buffer_to_update].state = kBufferSent;
 289       buffer_state_[buffer_to_update].frame_id = encoded_frame->frame_id;
 290     }
 291   }
 292 }
 293
 294 uint32 Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
 295   if (!use_multiple_video_buffers_)
 296     return last_encoded_frame_id_;
 297
 298   const uint32 kMagicFrameOffset = 512;
 299   // We set latest_frame_to_reference to an old frame so that
 300   // IsNewerFrameId will work correctly.
 301   uint32 latest_frame_to_reference =
 302       last_encoded_frame_id_ - kMagicFrameOffset;
 303
 304   // Reference all acked frames.
 305   // TODO(hubbe): We may also want to allow references to the
 306   // last encoded frame, if that frame was assigned to a buffer.
 307   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 308     if (buffer_state_[i].state == kBufferAcked) {
 309       if (IsNewerFrameId(buffer_state_[i].frame_id,
 310                          latest_frame_to_reference)) {
 311         latest_frame_to_reference = buffer_state_[i].frame_id;
 312       }
 313     } else {
 314       switch (i) {
 315         case kAltRefBuffer:
 316           *flags |= VP8_EFLAG_NO_REF_ARF;
 317           break;
 318         case kGoldenBuffer:
 319           *flags |= VP8_EFLAG_NO_REF_GF;
 320           break;
 321         case kLastBuffer:
 322           *flags |= VP8_EFLAG_NO_REF_LAST;
 323           break;
 324       }
 325     }
 326   }
 327
 328   if (latest_frame_to_reference ==
 329       last_encoded_frame_id_ - kMagicFrameOffset) {
 330     // We have nothing to reference, it's kind of like a key frame,
 331     // but doesn't reset buffers.
 332     latest_frame_to_reference = last_encoded_frame_id_ + 1;
 333   }
 334
 335   return latest_frame_to_reference;
 336 }
 337
 338 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() {
 339   if (!use_multiple_video_buffers_)
 340     return kNoBuffer;
 341
 342   // The goal here is to make sure that we always keep one ACKed
 343   // buffer while trying to get an ACK for a newer buffer as we go.
 344   // Here are the rules for which buffer to select for update:
 345   // 1. If there is a buffer in state kStartState, use it.
 346   // 2. If there is a buffer other than the oldest buffer
 347   //    which is Acked, use the oldest buffer.
 348   // 3. If there are Sent buffers which are older than
 349   //    latest_acked_frame_, use the oldest one.
 350   // 4. If all else fails, just overwrite the newest buffer,
 351   //    but no more than 3 times in a row.
 352   //    TODO(hubbe): Figure out if 3 is optimal.
 353   // Note, rule 1-3 describe cases where there is a "free" buffer
 354   // that we can use. Rule 4 describes what happens when there is
 355   // no free buffer available.
 356
 357   // Buffers, sorted from oldest frame to newest.
 358   Vp8Encoder::Vp8Buffers buffers[kNumberOfVp8VideoBuffers];
 359
 360   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 361     Vp8Encoder::Vp8Buffers buffer = static_cast<Vp8Encoder::Vp8Buffers>(i);
 362
 363     // Rule 1
 364     if (buffer_state_[buffer].state == kBufferStartState) {
 365       undroppable_frames_ = 0;
 366       return buffer;
 367     }
 368     buffers[buffer] = buffer;
 369   }
 370
 371   // Sorting three elements with selection sort.
 372   for (int i = 0; i < kNumberOfVp8VideoBuffers - 1; i++) {
 373     for (int j = i + 1; j < kNumberOfVp8VideoBuffers; j++) {
 374       if (IsOlderFrameId(buffer_state_[buffers[j]].frame_id,
 375                          buffer_state_[buffers[i]].frame_id)) {
 376         std::swap(buffers[i], buffers[j]);
 377       }
 378     }
 379   }
 380
 381   // Rule 2
 382   if (buffer_state_[buffers[1]].state == kBufferAcked ||
 383       buffer_state_[buffers[2]].state == kBufferAcked) {
 384     undroppable_frames_ = 0;
 385     return buffers[0];
 386   }
 387
 388   // Rule 3
 389   for (int i = 0; i < kNumberOfVp8VideoBuffers; i++) {
 390     if (buffer_state_[buffers[i]].state == kBufferSent &&
 391         IsOlderFrameId(buffer_state_[buffers[i]].frame_id,
 392                        last_acked_frame_id_)) {
 393       undroppable_frames_ = 0;
 394       return buffers[i];
 395     }
 396   }
 397
 398   // Rule 4
 399   if (undroppable_frames_ >= 3) {
 400     undroppable_frames_ = 0;
 401     return kNoBuffer;
 402   } else {
 403     undroppable_frames_++;
 404     return buffers[kNumberOfVp8VideoBuffers - 1];
 405   }
 406 }
 407
 408 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update,
 409                                      vpx_codec_flags_t* flags) {
 410   if (!use_multiple_video_buffers_)
 411     return;
 412
 413   // Update at most one buffer, except for key-frames.
 414   switch (buffer_to_update) {
 415     case kAltRefBuffer:
 416       *flags |= VP8_EFLAG_NO_UPD_GF;
 417       *flags |= VP8_EFLAG_NO_UPD_LAST;
 418       break;
 419     case kLastBuffer:
 420       *flags |= VP8_EFLAG_NO_UPD_GF;
 421       *flags |= VP8_EFLAG_NO_UPD_ARF;
 422       break;
 423     case kGoldenBuffer:
 424       *flags |= VP8_EFLAG_NO_UPD_ARF;
 425       *flags |= VP8_EFLAG_NO_UPD_LAST;
 426       break;
 427     case kNoBuffer:
 428       *flags |= VP8_EFLAG_NO_UPD_ARF;
 429       *flags |= VP8_EFLAG_NO_UPD_GF;
 430       *flags |= VP8_EFLAG_NO_UPD_LAST;
 431       *flags |= VP8_EFLAG_NO_UPD_ENTROPY;
 432       break;
 433   }
 434 }
 435
 436 void Vp8Encoder::UpdateRates(uint32 new_bitrate) {
 437   DCHECK(thread_checker_.CalledOnValidThread());
 438
 439   if (!is_initialized())
 440     return;
 441
 442   uint32 new_bitrate_kbit = new_bitrate / 1000;
 443   if (config_.rc_target_bitrate == new_bitrate_kbit)
 444     return;
 445
 446   config_.rc_target_bitrate = bitrate_kbit_ = new_bitrate_kbit;
 447
 448   // Update encoder context.
 449   if (vpx_codec_enc_config_set(&encoder_, &config_)) {
 450     NOTREACHED() << "Invalid return value";
 451   }
 452
 453   VLOG(1) << "VP8 new rc_target_bitrate: " << new_bitrate_kbit << " kbps";
 454 }
 455
 456 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
 457   DCHECK(thread_checker_.CalledOnValidThread());
 458   if (!use_multiple_video_buffers_)
 459     return;
 460
 461   VLOG(2) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
 462   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 463     if (frame_id == buffer_state_[i].frame_id) {
 464       buffer_state_[i].state = kBufferAcked;
 465       break;
 466     }
 467   }
 468   if (IsOlderFrameId(last_acked_frame_id_, frame_id)) {
 469     last_acked_frame_id_ = frame_id;
 470   }
 471 }
 472
 473 void Vp8Encoder::GenerateKeyFrame() {
 474   DCHECK(thread_checker_.CalledOnValidThread());
 475   key_frame_requested_ = true;
 476 }
 477
 478 }  // namespace cast
 479 }  // namespace media