media/cast/sender/vp8_encoder.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/vp8_encoder.h"
   6
   7 #include <vector>
   8
   9 #include "base/logging.h"
  10 #include "media/base/video_frame.h"
  11 #include "media/cast/cast_defines.h"
  12 #include "media/cast/net/cast_transport_config.h"
  13 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  14
  15 namespace media {
  16 namespace cast {
  17
  18 static const uint32 kMinIntra = 300;
  19
  20 static int ComputeMaxNumOfRepeatedBuffers(int max_unacked_frames) {
  21   if (max_unacked_frames > kNumberOfVp8VideoBuffers)
  22     return (max_unacked_frames - 1) / kNumberOfVp8VideoBuffers;
  23
  24   return 0;
  25 }
  26
  27 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config,
  28                        int max_unacked_frames)
  29     : cast_config_(video_config),
  30       use_multiple_video_buffers_(
  31           cast_config_.max_number_of_video_buffers_used ==
  32           kNumberOfVp8VideoBuffers),
  33       max_number_of_repeated_buffers_in_a_row_(
  34           ComputeMaxNumOfRepeatedBuffers(max_unacked_frames)),
  35       key_frame_requested_(true),
  36       first_frame_received_(false),
  37       last_encoded_frame_id_(kStartFrameId),
  38       number_of_repeated_buffers_(0) {
  39   // TODO(pwestin): we need to figure out how to synchronize the acking with the
  40   // internal state of the encoder, ideally the encoder will tell if we can
  41   // send another frame.
  42   DCHECK(!use_multiple_video_buffers_ ||
  43          max_number_of_repeated_buffers_in_a_row_ == 0)
  44       << "Invalid config";
  45
  46   // VP8 have 3 buffers available for prediction, with
  47   // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
  48   // however in this mode we can not skip frames in the receiver to catch up
  49   // after a temporary network outage; with max_number_of_video_buffers_used
  50   // set to 3 we allow 2 frames to be skipped by the receiver without error
  51   // propagation.
  52   DCHECK(cast_config_.max_number_of_video_buffers_used == 1 ||
  53          cast_config_.max_number_of_video_buffers_used ==
  54              kNumberOfVp8VideoBuffers)
  55       << "Invalid argument";
  56
  57   thread_checker_.DetachFromThread();
  58 }
  59
  60 Vp8Encoder::~Vp8Encoder() {
  61   vpx_codec_destroy(encoder_.get());
  62   vpx_img_free(raw_image_);
  63 }
  64
  65 void Vp8Encoder::Initialize() {
  66   DCHECK(thread_checker_.CalledOnValidThread());
  67   config_.reset(new vpx_codec_enc_cfg_t());
  68   encoder_.reset(new vpx_codec_ctx_t());
  69
  70   // Creating a wrapper to the image - setting image data to NULL. Actual
  71   // pointer will be set during encode. Setting align to 1, as it is
  72   // meaningless (actual memory is not allocated).
  73   raw_image_ = vpx_img_wrap(
  74       NULL, IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL);
  75
  76   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
  77     acked_frame_buffers_[i] = true;
  78     used_buffers_frame_id_[i] = kStartFrameId;
  79   }
  80   InitEncode(cast_config_.number_of_encode_threads);
  81 }
  82
  83 void Vp8Encoder::InitEncode(int number_of_encode_threads) {
  84   DCHECK(thread_checker_.CalledOnValidThread());
  85   // Populate encoder configuration with default values.
  86   if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), config_.get(), 0)) {
  87     DCHECK(false) << "Invalid return value";
  88   }
  89   config_->g_w = cast_config_.width;
  90   config_->g_h = cast_config_.height;
  91   config_->rc_target_bitrate = cast_config_.start_bitrate / 1000;  // In kbit/s.
  92
  93   // Setting the codec time base.
  94   config_->g_timebase.num = 1;
  95   config_->g_timebase.den = kVideoFrequency;
  96   config_->g_lag_in_frames = 0;
  97   config_->kf_mode = VPX_KF_DISABLED;
  98   if (use_multiple_video_buffers_) {
  99     // We must enable error resilience when we use multiple buffers, due to
 100     // codec requirements.
 101     config_->g_error_resilient = 1;
 102   }
 103   config_->g_threads = number_of_encode_threads;
 104
 105   // Rate control settings.
 106   // Never allow the encoder to drop frame internally.
 107   config_->rc_dropframe_thresh = 0;
 108   config_->rc_end_usage = VPX_CBR;
 109   config_->g_pass = VPX_RC_ONE_PASS;
 110   config_->rc_resize_allowed = 0;
 111   config_->rc_min_quantizer = cast_config_.min_qp;
 112   config_->rc_max_quantizer = cast_config_.max_qp;
 113   config_->rc_undershoot_pct = 100;
 114   config_->rc_overshoot_pct = 15;
 115   config_->rc_buf_initial_sz = 500;
 116   config_->rc_buf_optimal_sz = 600;
 117   config_->rc_buf_sz = 1000;
 118
 119   // set the maximum target size of any key-frame.
 120   uint32 rc_max_intra_target = MaxIntraTarget(config_->rc_buf_optimal_sz);
 121   vpx_codec_flags_t flags = 0;
 122   if (vpx_codec_enc_init(
 123           encoder_.get(), vpx_codec_vp8_cx(), config_.get(), flags)) {
 124     DCHECK(false) << "vpx_codec_enc_init() failed.";
 125     encoder_.reset();
 126     return;
 127   }
 128   vpx_codec_control(encoder_.get(), VP8E_SET_STATIC_THRESHOLD, 1);
 129   vpx_codec_control(encoder_.get(), VP8E_SET_NOISE_SENSITIVITY, 0);
 130   vpx_codec_control(encoder_.get(), VP8E_SET_CPUUSED, -6);
 131   vpx_codec_control(
 132       encoder_.get(), VP8E_SET_MAX_INTRA_BITRATE_PCT, rc_max_intra_target);
 133 }
 134
 135 bool Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame,
 136                         EncodedFrame* encoded_image) {
 137   DCHECK(thread_checker_.CalledOnValidThread());
 138   // Image in vpx_image_t format.
 139   // Input image is const. VP8's raw image is not defined as const.
 140   raw_image_->planes[PLANE_Y] =
 141       const_cast<uint8*>(video_frame->data(VideoFrame::kYPlane));
 142   raw_image_->planes[PLANE_U] =
 143       const_cast<uint8*>(video_frame->data(VideoFrame::kUPlane));
 144   raw_image_->planes[PLANE_V] =
 145       const_cast<uint8*>(video_frame->data(VideoFrame::kVPlane));
 146
 147   raw_image_->stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane);
 148   raw_image_->stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane);
 149   raw_image_->stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane);
 150
 151   uint8 latest_frame_id_to_reference;
 152   Vp8Buffers buffer_to_update;
 153   vpx_codec_flags_t flags = 0;
 154   if (key_frame_requested_) {
 155     flags = VPX_EFLAG_FORCE_KF;
 156     // Self reference.
 157     latest_frame_id_to_reference = last_encoded_frame_id_ + 1;
 158     // We can pick any buffer as buffer_to_update since we update
 159     // them all.
 160     buffer_to_update = kLastBuffer;
 161   } else {
 162     // Reference all acked frames (buffers).
 163     latest_frame_id_to_reference = GetLatestFrameIdToReference();
 164     GetCodecReferenceFlags(&flags);
 165     buffer_to_update = GetNextBufferToUpdate();
 166     GetCodecUpdateFlags(buffer_to_update, &flags);
 167   }
 168
 169   // Note: The duration does not reflect the real time between frames. This is
 170   // done to keep the encoder happy.
 171   //
 172   // TODO(miu): This is a semi-hack.  We should consider using
 173   // |video_frame->timestamp()| instead.
 174   uint32 duration = kVideoFrequency / cast_config_.max_frame_rate;
 175
 176   // Note: Timestamp here is used for bitrate calculation. The absolute value
 177   // is not important.
 178   if (!first_frame_received_) {
 179     first_frame_received_ = true;
 180     first_frame_timestamp_ = video_frame->timestamp();
 181   }
 182
 183   vpx_codec_pts_t timestamp =
 184       (video_frame->timestamp() - first_frame_timestamp_).InMicroseconds() *
 185       kVideoFrequency / base::Time::kMicrosecondsPerSecond;
 186
 187   if (vpx_codec_encode(encoder_.get(),
 188                        raw_image_,
 189                        timestamp,
 190                        duration,
 191                        flags,
 192                        VPX_DL_REALTIME) != VPX_CODEC_OK) {
 193     LOG(ERROR) << "Failed to encode for once.";
 194     return false;
 195   }
 196
 197   // Get encoded frame.
 198   const vpx_codec_cx_pkt_t* pkt = NULL;
 199   vpx_codec_iter_t iter = NULL;
 200   bool is_key_frame = false;
 201   while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) {
 202     if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
 203       continue;
 204     encoded_image->data.assign(
 205         static_cast<const uint8*>(pkt->data.frame.buf),
 206         static_cast<const uint8*>(pkt->data.frame.buf) + pkt->data.frame.sz);
 207     is_key_frame = !!(pkt->data.frame.flags & VPX_FRAME_IS_KEY);
 208     break;  // Done, since all data is provided in one CX_FRAME_PKT packet.
 209   }
 210   // Don't update frame_id for zero size frames.
 211   if (encoded_image->data.empty())
 212     return true;
 213
 214   // Populate the encoded frame.
 215   encoded_image->frame_id = ++last_encoded_frame_id_;
 216   if (is_key_frame) {
 217     encoded_image->dependency = EncodedFrame::KEY;
 218     encoded_image->referenced_frame_id = encoded_image->frame_id;
 219   } else {
 220     encoded_image->dependency = EncodedFrame::DEPENDENT;
 221     encoded_image->referenced_frame_id = latest_frame_id_to_reference;
 222   }
 223
 224   DVLOG(1) << "VP8 encoded frame_id " << encoded_image->frame_id
 225            << ", sized:" << encoded_image->data.size();
 226
 227   if (is_key_frame) {
 228     key_frame_requested_ = false;
 229
 230     for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 231       used_buffers_frame_id_[i] = encoded_image->frame_id;
 232     }
 233     // We can pick any buffer as last_used_vp8_buffer_ since we update
 234     // them all.
 235     last_used_vp8_buffer_ = buffer_to_update;
 236   } else {
 237     if (buffer_to_update != kNoBuffer) {
 238       acked_frame_buffers_[buffer_to_update] = false;
 239       used_buffers_frame_id_[buffer_to_update] = encoded_image->frame_id;
 240       last_used_vp8_buffer_ = buffer_to_update;
 241     }
 242   }
 243   return true;
 244 }
 245
 246 void Vp8Encoder::GetCodecReferenceFlags(vpx_codec_flags_t* flags) {
 247   if (!use_multiple_video_buffers_)
 248     return;
 249
 250   // We need to reference something.
 251   DCHECK(acked_frame_buffers_[kAltRefBuffer] ||
 252          acked_frame_buffers_[kGoldenBuffer] ||
 253          acked_frame_buffers_[kLastBuffer])
 254       << "Invalid state";
 255
 256   if (!acked_frame_buffers_[kAltRefBuffer]) {
 257     *flags |= VP8_EFLAG_NO_REF_ARF;
 258   }
 259   if (!acked_frame_buffers_[kGoldenBuffer]) {
 260     *flags |= VP8_EFLAG_NO_REF_GF;
 261   }
 262   if (!acked_frame_buffers_[kLastBuffer]) {
 263     *flags |= VP8_EFLAG_NO_REF_LAST;
 264   }
 265 }
 266
 267 uint32 Vp8Encoder::GetLatestFrameIdToReference() {
 268   if (!use_multiple_video_buffers_)
 269     return last_encoded_frame_id_;
 270
 271   int latest_frame_id_to_reference = -1;
 272   if (acked_frame_buffers_[kAltRefBuffer]) {
 273     latest_frame_id_to_reference = used_buffers_frame_id_[kAltRefBuffer];
 274   }
 275   if (acked_frame_buffers_[kGoldenBuffer]) {
 276     if (latest_frame_id_to_reference == -1) {
 277       latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
 278     } else {
 279       if (IsNewerFrameId(used_buffers_frame_id_[kGoldenBuffer],
 280                          latest_frame_id_to_reference)) {
 281         latest_frame_id_to_reference = used_buffers_frame_id_[kGoldenBuffer];
 282       }
 283     }
 284   }
 285   if (acked_frame_buffers_[kLastBuffer]) {
 286     if (latest_frame_id_to_reference == -1) {
 287       latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
 288     } else {
 289       if (IsNewerFrameId(used_buffers_frame_id_[kLastBuffer],
 290                          latest_frame_id_to_reference)) {
 291         latest_frame_id_to_reference = used_buffers_frame_id_[kLastBuffer];
 292       }
 293     }
 294   }
 295   DCHECK(latest_frame_id_to_reference != -1) << "Invalid state";
 296   return static_cast<uint32>(latest_frame_id_to_reference);
 297 }
 298
 299 Vp8Encoder::Vp8Buffers Vp8Encoder::GetNextBufferToUpdate() {
 300   if (!use_multiple_video_buffers_)
 301     return kNoBuffer;
 302
 303   // Update at most one buffer, except for key-frames.
 304
 305   Vp8Buffers buffer_to_update = kNoBuffer;
 306   if (number_of_repeated_buffers_ < max_number_of_repeated_buffers_in_a_row_) {
 307     // TODO(pwestin): experiment with this. The issue with only this change is
 308     // that we can end up with only 4 frames in flight when we expect 6.
 309     // buffer_to_update = last_used_vp8_buffer_;
 310     buffer_to_update = kNoBuffer;
 311     ++number_of_repeated_buffers_;
 312   } else {
 313     number_of_repeated_buffers_ = 0;
 314     switch (last_used_vp8_buffer_) {
 315       case kAltRefBuffer:
 316         buffer_to_update = kLastBuffer;
 317         VLOG(1) << "VP8 update last buffer";
 318         break;
 319       case kLastBuffer:
 320         buffer_to_update = kGoldenBuffer;
 321         VLOG(1) << "VP8 update golden buffer";
 322         break;
 323       case kGoldenBuffer:
 324         buffer_to_update = kAltRefBuffer;
 325         VLOG(1) << "VP8 update alt-ref buffer";
 326         break;
 327       case kNoBuffer:
 328         DCHECK(false) << "Invalid state";
 329         break;
 330     }
 331   }
 332   return buffer_to_update;
 333 }
 334
 335 void Vp8Encoder::GetCodecUpdateFlags(Vp8Buffers buffer_to_update,
 336                                      vpx_codec_flags_t* flags) {
 337   if (!use_multiple_video_buffers_)
 338     return;
 339
 340   // Update at most one buffer, except for key-frames.
 341   switch (buffer_to_update) {
 342     case kAltRefBuffer:
 343       *flags |= VP8_EFLAG_NO_UPD_GF;
 344       *flags |= VP8_EFLAG_NO_UPD_LAST;
 345       break;
 346     case kLastBuffer:
 347       *flags |= VP8_EFLAG_NO_UPD_GF;
 348       *flags |= VP8_EFLAG_NO_UPD_ARF;
 349       break;
 350     case kGoldenBuffer:
 351       *flags |= VP8_EFLAG_NO_UPD_ARF;
 352       *flags |= VP8_EFLAG_NO_UPD_LAST;
 353       break;
 354     case kNoBuffer:
 355       *flags |= VP8_EFLAG_NO_UPD_ARF;
 356       *flags |= VP8_EFLAG_NO_UPD_GF;
 357       *flags |= VP8_EFLAG_NO_UPD_LAST;
 358       *flags |= VP8_EFLAG_NO_UPD_ENTROPY;
 359       break;
 360   }
 361 }
 362
 363 void Vp8Encoder::UpdateRates(uint32 new_bitrate) {
 364   DCHECK(thread_checker_.CalledOnValidThread());
 365   uint32 new_bitrate_kbit = new_bitrate / 1000;
 366   if (config_->rc_target_bitrate == new_bitrate_kbit)
 367     return;
 368
 369   config_->rc_target_bitrate = new_bitrate_kbit;
 370
 371   // Update encoder context.
 372   if (vpx_codec_enc_config_set(encoder_.get(), config_.get())) {
 373     DCHECK(false) << "Invalid return value";
 374   }
 375 }
 376
 377 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
 378   DCHECK(thread_checker_.CalledOnValidThread());
 379   if (!use_multiple_video_buffers_)
 380     return;
 381
 382   VLOG(1) << "VP8 ok to reference frame:" << static_cast<int>(frame_id);
 383   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
 384     if (frame_id == used_buffers_frame_id_[i]) {
 385       acked_frame_buffers_[i] = true;
 386     }
 387   }
 388 }
 389
 390 void Vp8Encoder::GenerateKeyFrame() {
 391   DCHECK(thread_checker_.CalledOnValidThread());
 392   key_frame_requested_ = true;
 393 }
 394
 395 // Calculate the max size of the key frame relative to a normal delta frame.
 396 uint32 Vp8Encoder::MaxIntraTarget(uint32 optimal_buffer_size_ms) const {
 397   // Set max to the optimal buffer level (normalized by target BR),
 398   // and scaled by a scale_parameter.
 399   // Max target size = scalePar * optimalBufferSize * targetBR[Kbps].
 400   // This values is presented in percentage of perFrameBw:
 401   // perFrameBw = targetBR[Kbps] * 1000 / frameRate.
 402   // The target in % is as follows:
 403
 404   float scale_parameter = 0.5;
 405   uint32 target_pct = optimal_buffer_size_ms * scale_parameter *
 406                       cast_config_.max_frame_rate / 10;
 407
 408   // Don't go below 3 times the per frame bandwidth.
 409   return std::max(target_pct, kMinIntra);
 410 }
 411
 412 }  // namespace cast
 413 }  // namespace media