media/cast/sender/video_sender.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/video_sender.h"
   6
   7 #include <algorithm>
   8 #include <cstring>
   9
  10 #include "base/bind.h"
  11 #include "base/logging.h"
  12 #include "base/trace_event/trace_event.h"
  13 #include "media/cast/cast_defines.h"
  14 #include "media/cast/net/cast_transport_config.h"
  15 #include "media/cast/sender/performance_metrics_overlay.h"
  16 #include "media/cast/sender/video_encoder.h"
  17
  18 namespace media {
  19 namespace cast {
  20
  21 namespace {
  22
  23 // The following two constants are used to adjust the target
  24 // playout delay (when allowed). They were calculated using
  25 // a combination of cast_benchmark runs and manual testing.
  26 //
  27 // This is how many round trips we think we need on the network.
  28 const int kRoundTripsNeeded = 4;
  29
  30 // This is an estimate of all the the constant time needed independent of
  31 // network quality (e.g., additional time that accounts for encode and decode
  32 // time).
  33 const int kConstantTimeMs = 75;
  34
  35 // The target maximum utilization of the encoder and network resources.  This is
  36 // used to attenuate the actual measured utilization values in order to provide
  37 // "breathing room" (i.e., to ensure there will be sufficient CPU and bandwidth
  38 // available to handle the occasional more-complex frames).
  39 const int kTargetUtilizationPercentage = 75;
  40
  41 // Extract capture begin/end timestamps from |video_frame|'s metadata and log
  42 // it.
  43 void LogVideoCaptureTimestamps(const CastEnvironment& cast_environment,
  44                                const media::VideoFrame& video_frame,
  45                                RtpTimestamp rtp_timestamp) {
  46   base::TimeTicks capture_begin_time;
  47   base::TimeTicks capture_end_time;
  48   if (!video_frame.metadata()->GetTimeTicks(
  49           media::VideoFrameMetadata::CAPTURE_BEGIN_TIME, &capture_begin_time) ||
  50       !video_frame.metadata()->GetTimeTicks(
  51           media::VideoFrameMetadata::CAPTURE_END_TIME, &capture_end_time)) {
  52     // The frame capture timestamps were not provided by the video capture
  53     // source.  Simply log the events as happening right now.
  54     capture_begin_time = capture_end_time =
  55         cast_environment.Clock()->NowTicks();
  56   }
  57   cast_environment.Logging()->InsertFrameEvent(
  58       capture_begin_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, rtp_timestamp,
  59       kFrameIdUnknown);
  60   cast_environment.Logging()->InsertCapturedVideoFrameEvent(
  61       capture_end_time, rtp_timestamp, video_frame.visible_rect().width(),
  62       video_frame.visible_rect().height());
  63 }
  64
  65 }  // namespace
  66
  67 // Note, we use a fixed bitrate value when external video encoder is used.
  68 // Some hardware encoder shows bad behavior if we set the bitrate too
  69 // frequently, e.g. quality drop, not abiding by target bitrate, etc.
  70 // See details: crbug.com/392086.
  71 VideoSender::VideoSender(
  72     scoped_refptr<CastEnvironment> cast_environment,
  73     const VideoSenderConfig& video_config,
  74     const StatusChangeCallback& status_change_cb,
  75     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  76     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  77     CastTransportSender* const transport_sender,
  78     const PlayoutDelayChangeCB& playout_delay_change_cb)
  79     : FrameSender(
  80           cast_environment,
  81           false,
  82           transport_sender,
  83           kVideoFrequency,
  84           video_config.ssrc,
  85           video_config.max_frame_rate,
  86           video_config.min_playout_delay,
  87           video_config.max_playout_delay,
  88           video_config.use_external_encoder
  89               ? NewFixedCongestionControl(
  90                     (video_config.min_bitrate + video_config.max_bitrate) / 2)
  91               : NewAdaptiveCongestionControl(cast_environment->Clock(),
  92                                              video_config.max_bitrate,
  93                                              video_config.min_bitrate,
  94                                              video_config.max_frame_rate)),
  95       frames_in_encoder_(0),
  96       last_bitrate_(0),
  97       playout_delay_change_cb_(playout_delay_change_cb),
  98       last_reported_deadline_utilization_(-1.0),
  99       last_reported_lossy_utilization_(-1.0),
 100       weak_factory_(this) {
 101   video_encoder_ = VideoEncoder::Create(
 102       cast_environment_,
 103       video_config,
 104       status_change_cb,
 105       create_vea_cb,
 106       create_video_encode_mem_cb);
 107   if (!video_encoder_) {
 108     cast_environment_->PostTask(
 109         CastEnvironment::MAIN,
 110         FROM_HERE,
 111         base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC));
 112   }
 113
 114   media::cast::CastTransportRtpConfig transport_config;
 115   transport_config.ssrc = video_config.ssrc;
 116   transport_config.feedback_ssrc = video_config.receiver_ssrc;
 117   transport_config.rtp_payload_type = video_config.rtp_payload_type;
 118   transport_config.aes_key = video_config.aes_key;
 119   transport_config.aes_iv_mask = video_config.aes_iv_mask;
 120
 121   transport_sender->InitializeVideo(
 122       transport_config,
 123       base::Bind(&VideoSender::OnReceivedCastFeedback,
 124                  weak_factory_.GetWeakPtr()),
 125       base::Bind(&VideoSender::OnMeasuredRoundTripTime,
 126                  weak_factory_.GetWeakPtr()));
 127 }
 128
 129 VideoSender::~VideoSender() {
 130 }
 131
 132 void VideoSender::InsertRawVideoFrame(
 133     const scoped_refptr<media::VideoFrame>& video_frame,
 134     const base::TimeTicks& reference_time) {
 135   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 136
 137   if (!video_encoder_) {
 138     NOTREACHED();
 139     return;
 140   }
 141
 142   const RtpTimestamp rtp_timestamp =
 143       TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency);
 144   LogVideoCaptureTimestamps(*cast_environment_, *video_frame, rtp_timestamp);
 145
 146   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 147   TRACE_EVENT_INSTANT2(
 148       "cast_perf_test", "InsertRawVideoFrame",
 149       TRACE_EVENT_SCOPE_THREAD,
 150       "timestamp", reference_time.ToInternalValue(),
 151       "rtp_timestamp", rtp_timestamp);
 152
 153   // Drop the frame if either its RTP or reference timestamp is not an increase
 154   // over the last frame's.  This protects: 1) the duration calculations that
 155   // assume timestamps are monotonically non-decreasing, and 2) assumptions made
 156   // deeper in the implementation where each frame's RTP timestamp needs to be
 157   // unique.
 158   if (!last_enqueued_frame_reference_time_.is_null() &&
 159       (!IsNewerRtpTimestamp(rtp_timestamp,
 160                             last_enqueued_frame_rtp_timestamp_) ||
 161        reference_time <= last_enqueued_frame_reference_time_)) {
 162     VLOG(1) << "Dropping video frame: RTP or reference time did not increase.";
 163     return;
 164   }
 165
 166   // Two video frames are needed to compute the exact media duration added by
 167   // the next frame.  If there are no frames in the encoder, compute a guess
 168   // based on the configured |max_frame_rate_|.  Any error introduced by this
 169   // guess will be eliminated when |duration_in_encoder_| is updated in
 170   // OnEncodedVideoFrame().
 171   const base::TimeDelta duration_added_by_next_frame = frames_in_encoder_ > 0 ?
 172       reference_time - last_enqueued_frame_reference_time_ :
 173       base::TimeDelta::FromSecondsD(1.0 / max_frame_rate_);
 174
 175   if (ShouldDropNextFrame(duration_added_by_next_frame)) {
 176     base::TimeDelta new_target_delay = std::min(
 177         current_round_trip_time_ * kRoundTripsNeeded +
 178         base::TimeDelta::FromMilliseconds(kConstantTimeMs),
 179         max_playout_delay_);
 180     if (new_target_delay > target_playout_delay_) {
 181       VLOG(1) << "New target delay: " << new_target_delay.InMilliseconds();
 182       playout_delay_change_cb_.Run(new_target_delay);
 183     }
 184
 185     // Some encoder implementations have a frame window for analysis. Since we
 186     // are dropping this frame, unless we instruct the encoder to flush all the
 187     // frames that have been enqueued for encoding, frames_in_encoder_ and
 188     // last_enqueued_frame_reference_time_ will never be updated and we will
 189     // drop every subsequent frame for the rest of the session.
 190     video_encoder_->EmitFrames();
 191
 192     return;
 193   }
 194
 195   uint32 bitrate = congestion_control_->GetBitrate(
 196         reference_time + target_playout_delay_, target_playout_delay_);
 197   if (bitrate != last_bitrate_) {
 198     video_encoder_->SetBitRate(bitrate);
 199     last_bitrate_ = bitrate;
 200   }
 201
 202   if (video_frame->visible_rect().IsEmpty()) {
 203     VLOG(1) << "Rejecting empty video frame.";
 204     return;
 205   }
 206
 207   MaybeRenderPerformanceMetricsOverlay(bitrate,
 208                                        frames_in_encoder_ + 1,
 209                                        last_reported_deadline_utilization_,
 210                                        last_reported_lossy_utilization_,
 211                                        video_frame.get());
 212
 213   if (video_encoder_->EncodeVideoFrame(
 214           video_frame,
 215           reference_time,
 216           base::Bind(&VideoSender::OnEncodedVideoFrame,
 217                      weak_factory_.GetWeakPtr(),
 218                      video_frame,
 219                      bitrate))) {
 220     frames_in_encoder_++;
 221     duration_in_encoder_ += duration_added_by_next_frame;
 222     last_enqueued_frame_rtp_timestamp_ = rtp_timestamp;
 223     last_enqueued_frame_reference_time_ = reference_time;
 224   } else {
 225     VLOG(1) << "Encoder rejected a frame.  Skipping...";
 226   }
 227 }
 228
 229 scoped_ptr<VideoFrameFactory> VideoSender::CreateVideoFrameFactory() {
 230   return video_encoder_ ? video_encoder_->CreateVideoFrameFactory() : nullptr;
 231 }
 232
 233 int VideoSender::GetNumberOfFramesInEncoder() const {
 234   return frames_in_encoder_;
 235 }
 236
 237 base::TimeDelta VideoSender::GetInFlightMediaDuration() const {
 238   if (GetUnacknowledgedFrameCount() > 0) {
 239     const uint32 oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
 240     return last_enqueued_frame_reference_time_ -
 241         GetRecordedReferenceTime(oldest_unacked_frame_id);
 242   } else {
 243     return duration_in_encoder_;
 244   }
 245 }
 246
 247 void VideoSender::OnAck(uint32 frame_id) {
 248   video_encoder_->LatestFrameIdToReference(frame_id);
 249 }
 250
 251 void VideoSender::OnEncodedVideoFrame(
 252     const scoped_refptr<media::VideoFrame>& video_frame,
 253     int encoder_bitrate,
 254     scoped_ptr<SenderEncodedFrame> encoded_frame) {
 255   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 256
 257   frames_in_encoder_--;
 258   DCHECK_GE(frames_in_encoder_, 0);
 259
 260   duration_in_encoder_ =
 261       last_enqueued_frame_reference_time_ - encoded_frame->reference_time;
 262
 263   last_reported_deadline_utilization_ = encoded_frame->deadline_utilization;
 264   last_reported_lossy_utilization_ = encoded_frame->lossy_utilization;
 265
 266   // Report the resource utilization for processing this frame.  Take the
 267   // greater of the two utilization values and attenuate them such that the
 268   // target utilization is reported as the maximum sustainable amount.
 269   const double attenuated_utilization =
 270       std::max(last_reported_deadline_utilization_,
 271                last_reported_lossy_utilization_) /
 272           (kTargetUtilizationPercentage / 100.0);
 273   if (attenuated_utilization >= 0.0) {
 274     // Key frames are artificially capped to 1.0 because their actual
 275     // utilization is atypical compared to the other frames in the stream, and
 276     // this can misguide the producer of the input video frames.
 277     video_frame->metadata()->SetDouble(
 278         media::VideoFrameMetadata::RESOURCE_UTILIZATION,
 279         encoded_frame->dependency == EncodedFrame::KEY ?
 280             std::min(1.0, attenuated_utilization) : attenuated_utilization);
 281   }
 282
 283   SendEncodedFrame(encoder_bitrate, encoded_frame.Pass());
 284 }
 285
 286 }  // namespace cast
 287 }  // namespace media