media/cast/sender/video_sender.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/video_sender.h"
   6
   7 #include <algorithm>
   8 #include <cstring>
   9
  10 #include "base/bind.h"
  11 #include "base/debug/trace_event.h"
  12 #include "base/logging.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "media/cast/cast_defines.h"
  15 #include "media/cast/net/cast_transport_config.h"
  16 #include "media/cast/sender/external_video_encoder.h"
  17 #include "media/cast/sender/video_encoder_impl.h"
  18
  19 namespace media {
  20 namespace cast {
  21
  22 const int kNumAggressiveReportsSentAtStart = 100;
  23
  24 namespace {
  25
  26 // Returns a fixed bitrate value when external video encoder is used.
  27 // Some hardware encoder shows bad behavior if we set the bitrate too
  28 // frequently, e.g. quality drop, not abiding by target bitrate, etc.
  29 // See details: crbug.com/392086.
  30 size_t GetFixedBitrate(const VideoSenderConfig& video_config) {
  31   if (!video_config.use_external_encoder)
  32     return 0;
  33   return (video_config.min_bitrate + video_config.max_bitrate) / 2;
  34 }
  35
  36 }  // namespace
  37
  38 VideoSender::VideoSender(
  39     scoped_refptr<CastEnvironment> cast_environment,
  40     const VideoSenderConfig& video_config,
  41     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  42     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  43     CastTransportSender* const transport_sender)
  44     : FrameSender(
  45         cast_environment,
  46         transport_sender,
  47         base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
  48         kVideoFrequency,
  49         video_config.ssrc,
  50         video_config.max_frame_rate,
  51         video_config.target_playout_delay),
  52       fixed_bitrate_(GetFixedBitrate(video_config)),
  53       frames_in_encoder_(0),
  54       congestion_control_(cast_environment->Clock(),
  55                           video_config.max_bitrate,
  56                           video_config.min_bitrate,
  57                           max_unacked_frames_),
  58       weak_factory_(this) {
  59   cast_initialization_status_ = STATUS_VIDEO_UNINITIALIZED;
  60   VLOG(1) << "max_unacked_frames is " << max_unacked_frames_
  61           << " for target_playout_delay="
  62           << target_playout_delay_.InMilliseconds() << " ms"
  63           << " and max_frame_rate=" << video_config.max_frame_rate;
  64   DCHECK_GT(max_unacked_frames_, 0);
  65
  66   if (video_config.use_external_encoder) {
  67     video_encoder_.reset(new ExternalVideoEncoder(cast_environment,
  68                                                   video_config,
  69                                                   create_vea_cb,
  70                                                   create_video_encode_mem_cb));
  71   } else {
  72     video_encoder_.reset(new VideoEncoderImpl(
  73         cast_environment, video_config, max_unacked_frames_));
  74   }
  75   cast_initialization_status_ = STATUS_VIDEO_INITIALIZED;
  76
  77   media::cast::CastTransportRtpConfig transport_config;
  78   transport_config.ssrc = video_config.ssrc;
  79   transport_config.feedback_ssrc = video_config.incoming_feedback_ssrc;
  80   transport_config.rtp_payload_type = video_config.rtp_payload_type;
  81   transport_config.stored_frames = max_unacked_frames_;
  82   transport_config.aes_key = video_config.aes_key;
  83   transport_config.aes_iv_mask = video_config.aes_iv_mask;
  84
  85   transport_sender->InitializeVideo(
  86       transport_config,
  87       base::Bind(&VideoSender::OnReceivedCastFeedback,
  88                  weak_factory_.GetWeakPtr()),
  89       base::Bind(&VideoSender::OnReceivedRtt, weak_factory_.GetWeakPtr()));
  90 }
  91
  92 VideoSender::~VideoSender() {
  93 }
  94
  95 void VideoSender::InsertRawVideoFrame(
  96     const scoped_refptr<media::VideoFrame>& video_frame,
  97     const base::TimeTicks& capture_time) {
  98   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  99   if (cast_initialization_status_ != STATUS_VIDEO_INITIALIZED) {
 100     NOTREACHED();
 101     return;
 102   }
 103   DCHECK(video_encoder_.get()) << "Invalid state";
 104
 105   RtpTimestamp rtp_timestamp = GetVideoRtpTimestamp(capture_time);
 106   cast_environment_->Logging()->InsertFrameEvent(
 107       capture_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT,
 108       rtp_timestamp, kFrameIdUnknown);
 109   cast_environment_->Logging()->InsertFrameEvent(
 110       cast_environment_->Clock()->NowTicks(),
 111       FRAME_CAPTURE_END, VIDEO_EVENT,
 112       rtp_timestamp,
 113       kFrameIdUnknown);
 114
 115   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 116   TRACE_EVENT_INSTANT2(
 117       "cast_perf_test", "InsertRawVideoFrame",
 118       TRACE_EVENT_SCOPE_THREAD,
 119       "timestamp", capture_time.ToInternalValue(),
 120       "rtp_timestamp", rtp_timestamp);
 121
 122   if (ShouldDropNextFrame(capture_time)) {
 123     VLOG(1) << "Dropping frame due to too many frames currently in-flight.";
 124     return;
 125   }
 126
 127   uint32 bitrate = fixed_bitrate_;
 128   if (!bitrate) {
 129     bitrate = congestion_control_.GetBitrate(
 130         capture_time + target_playout_delay_, target_playout_delay_);
 131     DCHECK(bitrate);
 132     video_encoder_->SetBitRate(bitrate);
 133   } else if (last_send_time_.is_null()) {
 134     // Set the fixed bitrate value to codec until a frame is sent. We might
 135     // set this value a couple times at the very beginning of the stream but
 136     // it is not harmful.
 137     video_encoder_->SetBitRate(bitrate);
 138   }
 139
 140   if (video_encoder_->EncodeVideoFrame(
 141           video_frame,
 142           capture_time,
 143           base::Bind(&VideoSender::SendEncodedVideoFrame,
 144                      weak_factory_.GetWeakPtr(),
 145                      bitrate))) {
 146     frames_in_encoder_++;
 147   } else {
 148     VLOG(1) << "Encoder rejected a frame.  Skipping...";
 149   }
 150 }
 151
 152 void VideoSender::SendEncodedVideoFrame(
 153     int requested_bitrate_before_encode,
 154     scoped_ptr<EncodedFrame> encoded_frame) {
 155   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 156
 157   DCHECK_GT(frames_in_encoder_, 0);
 158   frames_in_encoder_--;
 159
 160   const uint32 frame_id = encoded_frame->frame_id;
 161
 162   const bool is_first_frame_to_be_sent = last_send_time_.is_null();
 163   last_send_time_ = cast_environment_->Clock()->NowTicks();
 164   last_sent_frame_id_ = frame_id;
 165   // If this is the first frame about to be sent, fake the value of
 166   // |latest_acked_frame_id_| to indicate the receiver starts out all caught up.
 167   // Also, schedule the periodic frame re-send checks.
 168   if (is_first_frame_to_be_sent) {
 169     latest_acked_frame_id_ = frame_id - 1;
 170     ScheduleNextResendCheck();
 171   }
 172
 173   VLOG_IF(1, encoded_frame->dependency == EncodedFrame::KEY)
 174       << "Send encoded key frame; frame_id: " << frame_id;
 175
 176   cast_environment_->Logging()->InsertEncodedFrameEvent(
 177       last_send_time_, FRAME_ENCODED, VIDEO_EVENT, encoded_frame->rtp_timestamp,
 178       frame_id, static_cast<int>(encoded_frame->data.size()),
 179       encoded_frame->dependency == EncodedFrame::KEY,
 180       requested_bitrate_before_encode);
 181
 182   RecordLatestFrameTimestamps(frame_id,
 183                               encoded_frame->reference_time,
 184                               encoded_frame->rtp_timestamp);
 185
 186   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 187   TRACE_EVENT_INSTANT1(
 188       "cast_perf_test", "VideoFrameEncoded",
 189       TRACE_EVENT_SCOPE_THREAD,
 190       "rtp_timestamp", encoded_frame->rtp_timestamp);
 191
 192   // At the start of the session, it's important to send reports before each
 193   // frame so that the receiver can properly compute playout times.  The reason
 194   // more than one report is sent is because transmission is not guaranteed,
 195   // only best effort, so send enough that one should almost certainly get
 196   // through.
 197   if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
 198     // SendRtcpReport() will schedule future reports to be made if this is the
 199     // last "aggressive report."
 200     ++num_aggressive_rtcp_reports_sent_;
 201     const bool is_last_aggressive_report =
 202         (num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart);
 203     VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report.";
 204     SendRtcpReport(is_last_aggressive_report);
 205   }
 206
 207   congestion_control_.SendFrameToTransport(
 208       frame_id, encoded_frame->data.size() * 8, last_send_time_);
 209
 210   if (send_target_playout_delay_) {
 211     encoded_frame->new_playout_delay_ms =
 212         target_playout_delay_.InMilliseconds();
 213   }
 214   transport_sender_->InsertCodedVideoFrame(*encoded_frame);
 215 }
 216
 217 void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
 218   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 219
 220   base::TimeDelta rtt;
 221   base::TimeDelta avg_rtt;
 222   base::TimeDelta min_rtt;
 223   base::TimeDelta max_rtt;
 224   if (is_rtt_available()) {
 225     rtt = rtt_;
 226     avg_rtt = avg_rtt_;
 227     min_rtt = min_rtt_;
 228     max_rtt = max_rtt_;
 229
 230     congestion_control_.UpdateRtt(rtt);
 231
 232     // Don't use a RTT lower than our average.
 233     rtt = std::max(rtt, avg_rtt);
 234
 235     // Having the RTT values implies the receiver sent back a receiver report
 236     // based on it having received a report from here.  Therefore, ensure this
 237     // sender stops aggressively sending reports.
 238     if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
 239       VLOG(1) << "No longer a need to send reports aggressively (sent "
 240               << num_aggressive_rtcp_reports_sent_ << ").";
 241       num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart;
 242       ScheduleNextRtcpReport();
 243     }
 244   } else {
 245     // We have no measured value use default.
 246     rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
 247   }
 248
 249   if (last_send_time_.is_null())
 250     return;  // Cannot get an ACK without having first sent a frame.
 251
 252   if (cast_feedback.missing_frames_and_packets.empty()) {
 253     video_encoder_->LatestFrameIdToReference(cast_feedback.ack_frame_id);
 254
 255     // We only count duplicate ACKs when we have sent newer frames.
 256     if (latest_acked_frame_id_ == cast_feedback.ack_frame_id &&
 257         latest_acked_frame_id_ != last_sent_frame_id_) {
 258       duplicate_ack_counter_++;
 259     } else {
 260       duplicate_ack_counter_ = 0;
 261     }
 262     // TODO(miu): The values "2" and "3" should be derived from configuration.
 263     if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) {
 264       VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_;
 265       ResendForKickstart();
 266     }
 267   } else {
 268     // Only count duplicated ACKs if there is no NACK request in between.
 269     // This is to avoid aggresive resend.
 270     duplicate_ack_counter_ = 0;
 271   }
 272
 273   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 274   congestion_control_.AckFrame(cast_feedback.ack_frame_id, now);
 275
 276   cast_environment_->Logging()->InsertFrameEvent(
 277       now,
 278       FRAME_ACK_RECEIVED,
 279       VIDEO_EVENT,
 280       GetRecordedRtpTimestamp(cast_feedback.ack_frame_id),
 281       cast_feedback.ack_frame_id);
 282
 283   const bool is_acked_out_of_order =
 284       static_cast<int32>(cast_feedback.ack_frame_id -
 285                              latest_acked_frame_id_) < 0;
 286   VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "")
 287           << " for frame " << cast_feedback.ack_frame_id;
 288   if (!is_acked_out_of_order) {
 289     // Cancel resends of acked frames.
 290     std::vector<uint32> cancel_sending_frames;
 291     while (latest_acked_frame_id_ != cast_feedback.ack_frame_id) {
 292       latest_acked_frame_id_++;
 293       cancel_sending_frames.push_back(latest_acked_frame_id_);
 294     }
 295     transport_sender_->CancelSendingFrames(ssrc_, cancel_sending_frames);
 296     latest_acked_frame_id_ = cast_feedback.ack_frame_id;
 297   }
 298 }
 299
 300 bool VideoSender::ShouldDropNextFrame(base::TimeTicks capture_time) const {
 301   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 302   int frames_in_flight = 0;
 303   base::TimeDelta duration_in_flight;
 304   if (!last_send_time_.is_null()) {
 305     frames_in_flight =
 306         static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_);
 307     if (frames_in_flight > 0) {
 308       const uint32 oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
 309       duration_in_flight =
 310           capture_time - GetRecordedReferenceTime(oldest_unacked_frame_id);
 311     }
 312   }
 313   frames_in_flight += frames_in_encoder_;
 314   VLOG(2) << frames_in_flight
 315           << " frames in flight; last sent: " << last_sent_frame_id_
 316           << "; latest acked: " << latest_acked_frame_id_
 317           << "; frames in encoder: " << frames_in_encoder_
 318           << "; duration in flight: "
 319           << duration_in_flight.InMicroseconds() << " usec ("
 320           << (target_playout_delay_ > base::TimeDelta() ?
 321                   100 * duration_in_flight / target_playout_delay_ :
 322                   kint64max) << "%)";
 323   return frames_in_flight >= max_unacked_frames_ ||
 324       duration_in_flight >= target_playout_delay_;
 325 }
 326
 327 }  // namespace cast
 328 }  // namespace media