media/cast/sender/video_sender.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/sender/video_sender.h"
   6
   7 #include <algorithm>
   8 #include <cstring>
   9
  10 #include "base/bind.h"
  11 #include "base/logging.h"
  12 #include "base/trace_event/trace_event.h"
  13 #include "media/cast/cast_defines.h"
  14 #include "media/cast/net/cast_transport_config.h"
  15 #include "media/cast/sender/video_encoder.h"
  16
  17 namespace media {
  18 namespace cast {
  19
  20 namespace {
  21
  22 // The following two constants are used to adjust the target
  23 // playout delay (when allowed). They were calculated using
  24 // a combination of cast_benchmark runs and manual testing.
  25 //
  26 // This is how many round trips we think we need on the network.
  27 const int kRoundTripsNeeded = 4;
  28 // This is an estimate of all the the constant time needed independent of
  29 // network quality (e.g., additional time that accounts for encode and decode
  30 // time).
  31 const int kConstantTimeMs = 75;
  32
  33 }  // namespace
  34
  35 // Note, we use a fixed bitrate value when external video encoder is used.
  36 // Some hardware encoder shows bad behavior if we set the bitrate too
  37 // frequently, e.g. quality drop, not abiding by target bitrate, etc.
  38 // See details: crbug.com/392086.
  39 VideoSender::VideoSender(
  40     scoped_refptr<CastEnvironment> cast_environment,
  41     const VideoSenderConfig& video_config,
  42     const StatusChangeCallback& status_change_cb,
  43     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  44     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  45     CastTransportSender* const transport_sender,
  46     const PlayoutDelayChangeCB& playout_delay_change_cb)
  47     : FrameSender(
  48         cast_environment,
  49         false,
  50         transport_sender,
  51         base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
  52         kVideoFrequency,
  53         video_config.ssrc,
  54         video_config.max_frame_rate,
  55         video_config.min_playout_delay,
  56         video_config.max_playout_delay,
  57         video_config.use_external_encoder ?
  58             NewFixedCongestionControl(
  59                 (video_config.min_bitrate + video_config.max_bitrate) / 2) :
  60             NewAdaptiveCongestionControl(cast_environment->Clock(),
  61                                          video_config.max_bitrate,
  62                                          video_config.min_bitrate,
  63                                          video_config.max_frame_rate)),
  64       frames_in_encoder_(0),
  65       last_bitrate_(0),
  66       playout_delay_change_cb_(playout_delay_change_cb),
  67       weak_factory_(this) {
  68   video_encoder_ = VideoEncoder::Create(
  69       cast_environment_,
  70       video_config,
  71       status_change_cb,
  72       create_vea_cb,
  73       create_video_encode_mem_cb);
  74   if (!video_encoder_) {
  75     cast_environment_->PostTask(
  76         CastEnvironment::MAIN,
  77         FROM_HERE,
  78         base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC));
  79   }
  80
  81   media::cast::CastTransportRtpConfig transport_config;
  82   transport_config.ssrc = video_config.ssrc;
  83   transport_config.feedback_ssrc = video_config.receiver_ssrc;
  84   transport_config.rtp_payload_type = video_config.rtp_payload_type;
  85   transport_config.aes_key = video_config.aes_key;
  86   transport_config.aes_iv_mask = video_config.aes_iv_mask;
  87
  88   transport_sender->InitializeVideo(
  89       transport_config,
  90       base::Bind(&VideoSender::OnReceivedCastFeedback,
  91                  weak_factory_.GetWeakPtr()),
  92       base::Bind(&VideoSender::OnMeasuredRoundTripTime,
  93                  weak_factory_.GetWeakPtr()));
  94 }
  95
  96 VideoSender::~VideoSender() {
  97 }
  98
  99 void VideoSender::InsertRawVideoFrame(
 100     const scoped_refptr<media::VideoFrame>& video_frame,
 101     const base::TimeTicks& reference_time) {
 102   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 103
 104   if (!video_encoder_) {
 105     NOTREACHED();
 106     return;
 107   }
 108
 109   const RtpTimestamp rtp_timestamp =
 110       TimeDeltaToRtpDelta(video_frame->timestamp(), kVideoFrequency);
 111   const base::TimeTicks insertion_time = cast_environment_->Clock()->NowTicks();
 112   // TODO(miu): Plumb in capture timestamps.  For now, make it look like capture
 113   // took zero time by setting the BEGIN and END event to the same timestamp.
 114   cast_environment_->Logging()->InsertFrameEvent(
 115       insertion_time, FRAME_CAPTURE_BEGIN, VIDEO_EVENT, rtp_timestamp,
 116       kFrameIdUnknown);
 117   cast_environment_->Logging()->InsertFrameEvent(
 118       insertion_time, FRAME_CAPTURE_END, VIDEO_EVENT, rtp_timestamp,
 119       kFrameIdUnknown);
 120
 121   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 122   TRACE_EVENT_INSTANT2(
 123       "cast_perf_test", "InsertRawVideoFrame",
 124       TRACE_EVENT_SCOPE_THREAD,
 125       "timestamp", reference_time.ToInternalValue(),
 126       "rtp_timestamp", rtp_timestamp);
 127
 128   // Drop the frame if either its RTP or reference timestamp is not an increase
 129   // over the last frame's.  This protects: 1) the duration calculations that
 130   // assume timestamps are monotonically non-decreasing, and 2) assumptions made
 131   // deeper in the implementation where each frame's RTP timestamp needs to be
 132   // unique.
 133   if (!last_enqueued_frame_reference_time_.is_null() &&
 134       (!IsNewerRtpTimestamp(rtp_timestamp,
 135                             last_enqueued_frame_rtp_timestamp_) ||
 136        reference_time <= last_enqueued_frame_reference_time_)) {
 137     VLOG(1) << "Dropping video frame: RTP or reference time did not increase.";
 138     return;
 139   }
 140
 141   // Two video frames are needed to compute the exact media duration added by
 142   // the next frame.  If there are no frames in the encoder, compute a guess
 143   // based on the configured |max_frame_rate_|.  Any error introduced by this
 144   // guess will be eliminated when |duration_in_encoder_| is updated in
 145   // OnEncodedVideoFrame().
 146   const base::TimeDelta duration_added_by_next_frame = frames_in_encoder_ > 0 ?
 147       reference_time - last_enqueued_frame_reference_time_ :
 148       base::TimeDelta::FromSecondsD(1.0 / max_frame_rate_);
 149
 150   if (ShouldDropNextFrame(duration_added_by_next_frame)) {
 151     base::TimeDelta new_target_delay = std::min(
 152         current_round_trip_time_ * kRoundTripsNeeded +
 153         base::TimeDelta::FromMilliseconds(kConstantTimeMs),
 154         max_playout_delay_);
 155     if (new_target_delay > target_playout_delay_) {
 156       VLOG(1) << "New target delay: " << new_target_delay.InMilliseconds();
 157       playout_delay_change_cb_.Run(new_target_delay);
 158     }
 159
 160     // Some encoder implementations have a frame window for analysis. Since we
 161     // are dropping this frame, unless we instruct the encoder to flush all the
 162     // frames that have been enqueued for encoding, frames_in_encoder_ and
 163     // last_enqueued_frame_reference_time_ will never be updated and we will
 164     // drop every subsequent frame for the rest of the session.
 165     video_encoder_->EmitFrames();
 166
 167     return;
 168   }
 169
 170   uint32 bitrate = congestion_control_->GetBitrate(
 171         reference_time + target_playout_delay_, target_playout_delay_);
 172   if (bitrate != last_bitrate_) {
 173     video_encoder_->SetBitRate(bitrate);
 174     last_bitrate_ = bitrate;
 175   }
 176
 177   if (video_frame->visible_rect().IsEmpty()) {
 178     VLOG(1) << "Rejecting empty video frame.";
 179     return;
 180   }
 181
 182   if (video_encoder_->EncodeVideoFrame(
 183           video_frame,
 184           reference_time,
 185           base::Bind(&VideoSender::OnEncodedVideoFrame,
 186                      weak_factory_.GetWeakPtr(),
 187                      bitrate))) {
 188     frames_in_encoder_++;
 189     duration_in_encoder_ += duration_added_by_next_frame;
 190     last_enqueued_frame_rtp_timestamp_ = rtp_timestamp;
 191     last_enqueued_frame_reference_time_ = reference_time;
 192   } else {
 193     VLOG(1) << "Encoder rejected a frame.  Skipping...";
 194   }
 195 }
 196
 197 scoped_ptr<VideoFrameFactory> VideoSender::CreateVideoFrameFactory() {
 198   return video_encoder_ ? video_encoder_->CreateVideoFrameFactory() : nullptr;
 199 }
 200
 201 int VideoSender::GetNumberOfFramesInEncoder() const {
 202   return frames_in_encoder_;
 203 }
 204
 205 base::TimeDelta VideoSender::GetInFlightMediaDuration() const {
 206   if (GetUnacknowledgedFrameCount() > 0) {
 207     const uint32 oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
 208     return last_enqueued_frame_reference_time_ -
 209         GetRecordedReferenceTime(oldest_unacked_frame_id);
 210   } else {
 211     return duration_in_encoder_;
 212   }
 213 }
 214
 215 void VideoSender::OnAck(uint32 frame_id) {
 216   video_encoder_->LatestFrameIdToReference(frame_id);
 217 }
 218
 219 void VideoSender::OnEncodedVideoFrame(
 220     int encoder_bitrate,
 221     scoped_ptr<EncodedFrame> encoded_frame) {
 222   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 223
 224   frames_in_encoder_--;
 225   DCHECK_GE(frames_in_encoder_, 0);
 226
 227   duration_in_encoder_ =
 228       last_enqueued_frame_reference_time_ - encoded_frame->reference_time;
 229
 230   SendEncodedFrame(encoder_bitrate, encoded_frame.Pass());
 231 }
 232
 233 }  // namespace cast
 234 }  // namespace media