media/cast/video_sender/video_sender.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_sender/video_sender.h"
   6
   7 #include <cstring>
   8 #include <list>
   9
  10 #include "base/bind.h"
  11 #include "base/debug/trace_event.h"
  12 #include "base/logging.h"
  13 #include "base/message_loop/message_loop.h"
  14 #include "media/cast/cast_defines.h"
  15 #include "media/cast/rtcp/rtcp_defines.h"
  16 #include "media/cast/rtcp/sender_rtcp_event_subscriber.h"
  17 #include "media/cast/transport/cast_transport_config.h"
  18 #include "media/cast/video_sender/external_video_encoder.h"
  19 #include "media/cast/video_sender/video_encoder_impl.h"
  20
  21 namespace media {
  22 namespace cast {
  23
  24 const int64 kMinSchedulingDelayMs = 1;
  25
  26 // This is the maxmimum number of sender frame log messages that can fit in a
  27 // single RTCP packet.
  28 const int64 kMaxEventSubscriberEntries =
  29     (kMaxIpPacketSize - kRtcpCastLogHeaderSize) / kRtcpSenderFrameLogSize;
  30
  31 class LocalRtcpVideoSenderFeedback : public RtcpSenderFeedback {
  32  public:
  33   explicit LocalRtcpVideoSenderFeedback(VideoSender* video_sender)
  34       : video_sender_(video_sender) {}
  35
  36   virtual void OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback)
  37       OVERRIDE {
  38     video_sender_->OnReceivedCastFeedback(cast_feedback);
  39   }
  40
  41  private:
  42   VideoSender* video_sender_;
  43
  44   DISALLOW_IMPLICIT_CONSTRUCTORS(LocalRtcpVideoSenderFeedback);
  45 };
  46
  47 VideoSender::VideoSender(
  48     scoped_refptr<CastEnvironment> cast_environment,
  49     const VideoSenderConfig& video_config,
  50     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  51     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
  52     const CastInitializationCallback& cast_initialization_cb,
  53     transport::CastTransportSender* const transport_sender)
  54     : rtp_max_delay_(base::TimeDelta::FromMilliseconds(
  55           video_config.rtp_config.max_delay_ms)),
  56       max_frame_rate_(video_config.max_frame_rate),
  57       cast_environment_(cast_environment),
  58       transport_sender_(transport_sender),
  59       event_subscriber_(kMaxEventSubscriberEntries),
  60       rtp_stats_(kVideoFrequency),
  61       rtcp_feedback_(new LocalRtcpVideoSenderFeedback(this)),
  62       last_acked_frame_id_(-1),
  63       last_sent_frame_id_(-1),
  64       frames_in_encoder_(0),
  65       duplicate_ack_(0),
  66       last_skip_count_(0),
  67       current_requested_bitrate_(video_config.start_bitrate),
  68       congestion_control_(cast_environment->Clock(),
  69                           video_config.congestion_control_back_off,
  70                           video_config.max_bitrate,
  71                           video_config.min_bitrate,
  72                           video_config.start_bitrate),
  73       initialized_(false),
  74       active_session_(false),
  75       weak_factory_(this) {
  76   max_unacked_frames_ =
  77       1 + static_cast<uint8>(video_config.rtp_config.max_delay_ms *
  78                              max_frame_rate_ / 1000);
  79   VLOG(1) << "max_unacked_frames " << static_cast<int>(max_unacked_frames_);
  80   DCHECK_GT(max_unacked_frames_, 0) << "Invalid argument";
  81
  82   if (video_config.use_external_encoder) {
  83     video_encoder_.reset(new ExternalVideoEncoder(cast_environment,
  84                                                   video_config,
  85                                                   create_vea_cb,
  86                                                   create_video_encode_mem_cb));
  87   } else {
  88     video_encoder_.reset(new VideoEncoderImpl(
  89         cast_environment, video_config, max_unacked_frames_));
  90   }
  91
  92
  93   media::cast::transport::CastTransportVideoConfig transport_config;
  94   transport_config.codec = video_config.codec;
  95   transport_config.rtp.config = video_config.rtp_config;
  96   transport_config.rtp.max_outstanding_frames = max_unacked_frames_ + 1;
  97   transport_sender_->InitializeVideo(transport_config);
  98
  99   rtcp_.reset(
 100       new Rtcp(cast_environment_,
 101                rtcp_feedback_.get(),
 102                transport_sender_,
 103                NULL,  // paced sender.
 104                NULL,
 105                video_config.rtcp_mode,
 106                base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
 107                video_config.rtp_config.ssrc,
 108                video_config.incoming_feedback_ssrc,
 109                video_config.rtcp_c_name));
 110   rtcp_->SetCastReceiverEventHistorySize(kReceiverRtcpEventHistorySize);
 111
 112   // TODO(pwestin): pass cast_initialization_cb to |video_encoder_|
 113   // and remove this call.
 114   cast_environment_->PostTask(
 115       CastEnvironment::MAIN,
 116       FROM_HERE,
 117       base::Bind(cast_initialization_cb, STATUS_VIDEO_INITIALIZED));
 118   cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_);
 119
 120   memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_));
 121
 122   transport_sender_->SubscribeVideoRtpStatsCallback(
 123       base::Bind(&VideoSender::StoreStatistics, weak_factory_.GetWeakPtr()));
 124 }
 125
 126 VideoSender::~VideoSender() {
 127   cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_);
 128 }
 129
 130 void VideoSender::InitializeTimers() {
 131   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 132   if (!initialized_) {
 133     initialized_ = true;
 134     ScheduleNextResendCheck();
 135     ScheduleNextSkippedFramesCheck();
 136   }
 137 }
 138
 139 void VideoSender::InsertRawVideoFrame(
 140     const scoped_refptr<media::VideoFrame>& video_frame,
 141     const base::TimeTicks& capture_time) {
 142   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 143   DCHECK(video_encoder_.get()) << "Invalid state";
 144
 145   RtpTimestamp rtp_timestamp = GetVideoRtpTimestamp(capture_time);
 146   cast_environment_->Logging()->InsertFrameEvent(
 147       capture_time, kVideoFrameCaptureBegin, rtp_timestamp, kFrameIdUnknown);
 148   cast_environment_->Logging()->InsertFrameEvent(
 149       cast_environment_->Clock()->NowTicks(),
 150       kVideoFrameCaptureEnd,
 151       rtp_timestamp,
 152       kFrameIdUnknown);
 153
 154   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 155   TRACE_EVENT_INSTANT2(
 156       "cast_perf_test", "InsertRawVideoFrame",
 157       TRACE_EVENT_SCOPE_THREAD,
 158       "timestamp", capture_time.ToInternalValue(),
 159       "rtp_timestamp", GetVideoRtpTimestamp(capture_time));
 160
 161   if (video_encoder_->EncodeVideoFrame(
 162           video_frame,
 163           capture_time,
 164           base::Bind(&VideoSender::SendEncodedVideoFrameMainThread,
 165                      weak_factory_.GetWeakPtr()))) {
 166     frames_in_encoder_++;
 167     UpdateFramesInFlight();
 168   }
 169 }
 170
 171 void VideoSender::SendEncodedVideoFrameMainThread(
 172     scoped_ptr<transport::EncodedVideoFrame> encoded_frame,
 173     const base::TimeTicks& capture_time) {
 174   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 175   last_send_time_ = cast_environment_->Clock()->NowTicks();
 176   if (encoded_frame->key_frame) {
 177     VLOG(1) << "Send encoded key frame; frame_id:"
 178             << static_cast<int>(encoded_frame->frame_id);
 179   }
 180
 181   DCHECK_GT(frames_in_encoder_, 0);
 182   frames_in_encoder_--;
 183   uint32 frame_id = encoded_frame->frame_id;
 184   cast_environment_->Logging()->InsertEncodedFrameEvent(
 185       last_send_time_, kVideoFrameEncoded, encoded_frame->rtp_timestamp,
 186       frame_id, static_cast<int>(encoded_frame->data.size()),
 187       encoded_frame->key_frame, current_requested_bitrate_);
 188
 189   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
 190   TRACE_EVENT_INSTANT1(
 191       "cast_perf_test", "VideoFrameEncoded",
 192       TRACE_EVENT_SCOPE_THREAD,
 193       "rtp_timestamp", GetVideoRtpTimestamp(capture_time));
 194
 195   // Only use lowest 8 bits as key.
 196   frame_id_to_rtp_timestamp_[frame_id & 0xff] = encoded_frame->rtp_timestamp;
 197
 198   last_sent_frame_id_ = static_cast<int>(encoded_frame->frame_id);
 199   transport_sender_->InsertCodedVideoFrame(encoded_frame.get(), capture_time);
 200   UpdateFramesInFlight();
 201   InitializeTimers();
 202 }
 203
 204 void VideoSender::IncomingRtcpPacket(scoped_ptr<Packet> packet) {
 205   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 206   rtcp_->IncomingRtcpPacket(&packet->front(), packet->size());
 207 }
 208
 209 void VideoSender::ScheduleNextRtcpReport() {
 210   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 211   base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
 212                                  cast_environment_->Clock()->NowTicks();
 213
 214   time_to_next = std::max(
 215       time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 216
 217   cast_environment_->PostDelayedTask(
 218       CastEnvironment::MAIN,
 219       FROM_HERE,
 220       base::Bind(&VideoSender::SendRtcpReport, weak_factory_.GetWeakPtr()),
 221       time_to_next);
 222 }
 223
 224 void VideoSender::StoreStatistics(
 225     const transport::RtcpSenderInfo& sender_info,
 226     base::TimeTicks time_sent,
 227     uint32 rtp_timestamp) {
 228   rtp_stats_.Store(sender_info, time_sent, rtp_timestamp);
 229 }
 230
 231 void VideoSender::SendRtcpReport() {
 232   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 233
 234   transport::RtcpSenderLogMessage sender_log_message;
 235   RtcpEventMap rtcp_events;
 236   event_subscriber_.GetRtcpEventsAndReset(&rtcp_events);
 237
 238   for (RtcpEventMap::iterator it = rtcp_events.begin(); it != rtcp_events.end();
 239        ++it) {
 240     CastLoggingEvent event_type = it->second.type;
 241     if (event_type == kVideoFrameCaptureBegin ||
 242         event_type == kVideoFrameSentToEncoder ||
 243         event_type == kVideoFrameEncoded) {
 244       transport::RtcpSenderFrameLogMessage frame_message;
 245       frame_message.rtp_timestamp = it->first;
 246       switch (event_type) {
 247         case kVideoFrameCaptureBegin:
 248           frame_message.frame_status =
 249               transport::kRtcpSenderFrameStatusDroppedByFlowControl;
 250           break;
 251         case kVideoFrameSentToEncoder:
 252           frame_message.frame_status =
 253               transport::kRtcpSenderFrameStatusDroppedByEncoder;
 254           break;
 255         case kVideoFrameEncoded:
 256           frame_message.frame_status =
 257               transport::kRtcpSenderFrameStatusSentToNetwork;
 258           break;
 259         default:
 260           NOTREACHED();
 261           break;
 262       }
 263       sender_log_message.push_back(frame_message);
 264     } else {
 265       // This shouldn't happen because RtcpEventMap isn't supposed to contain
 266       // other event types.
 267       NOTREACHED() << "Got unknown event type in RtcpEventMap: " << event_type;
 268     }
 269   }
 270
 271   rtp_stats_.UpdateInfo(cast_environment_->Clock()->NowTicks());
 272
 273   rtcp_->SendRtcpFromRtpSender(sender_log_message, rtp_stats_.sender_info());
 274   if (!sender_log_message.empty()) {
 275     VLOG(1) << "Failed to send all log messages";
 276   }
 277   ScheduleNextRtcpReport();
 278 }
 279
 280 void VideoSender::ScheduleNextResendCheck() {
 281   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 282   base::TimeDelta time_to_next;
 283   if (last_send_time_.is_null()) {
 284     time_to_next = rtp_max_delay_;
 285   } else {
 286     time_to_next = last_send_time_ - cast_environment_->Clock()->NowTicks() +
 287                    rtp_max_delay_;
 288   }
 289   time_to_next = std::max(
 290       time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 291
 292   cast_environment_->PostDelayedTask(
 293       CastEnvironment::MAIN,
 294       FROM_HERE,
 295       base::Bind(&VideoSender::ResendCheck, weak_factory_.GetWeakPtr()),
 296       time_to_next);
 297 }
 298
 299 void VideoSender::ResendCheck() {
 300   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 301   if (!last_send_time_.is_null() && last_sent_frame_id_ != -1) {
 302     base::TimeDelta time_since_last_send =
 303         cast_environment_->Clock()->NowTicks() - last_send_time_;
 304     if (time_since_last_send > rtp_max_delay_) {
 305       if (!active_session_) {
 306         // We have not received any acks, resend the first encoded frame (id 0),
 307         // which must also be a key frame.
 308         VLOG(1) << "ACK timeout resend first key frame";
 309         ResendFrame(0);
 310       } else {
 311         if (last_acked_frame_id_ == last_sent_frame_id_) {
 312           // Last frame acked, no point in doing anything
 313         } else {
 314           DCHECK_LE(0, last_acked_frame_id_);
 315           uint32 frame_id = static_cast<uint32>(last_acked_frame_id_ + 1);
 316           VLOG(1) << "ACK timeout resend frame:" << static_cast<int>(frame_id);
 317           ResendFrame(frame_id);
 318         }
 319       }
 320     }
 321   }
 322   ScheduleNextResendCheck();
 323 }
 324
 325 void VideoSender::ScheduleNextSkippedFramesCheck() {
 326   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 327   base::TimeDelta time_to_next;
 328   if (last_checked_skip_count_time_.is_null()) {
 329     time_to_next =
 330         base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
 331   } else {
 332     time_to_next =
 333         last_checked_skip_count_time_ - cast_environment_->Clock()->NowTicks() +
 334         base::TimeDelta::FromMilliseconds(kSkippedFramesCheckPeriodkMs);
 335   }
 336   time_to_next = std::max(
 337       time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 338
 339   cast_environment_->PostDelayedTask(
 340       CastEnvironment::MAIN,
 341       FROM_HERE,
 342       base::Bind(&VideoSender::SkippedFramesCheck, weak_factory_.GetWeakPtr()),
 343       time_to_next);
 344 }
 345
 346 void VideoSender::SkippedFramesCheck() {
 347   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 348   int skip_count = video_encoder_->NumberOfSkippedFrames();
 349   if (skip_count - last_skip_count_ >
 350       kSkippedFramesThreshold * max_frame_rate_) {
 351     // TODO(pwestin): Propagate this up to the application.
 352   }
 353   last_skip_count_ = skip_count;
 354   last_checked_skip_count_time_ = cast_environment_->Clock()->NowTicks();
 355   ScheduleNextSkippedFramesCheck();
 356 }
 357
 358 void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
 359   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 360   base::TimeDelta rtt;
 361   base::TimeDelta avg_rtt;
 362   base::TimeDelta min_rtt;
 363   base::TimeDelta max_rtt;
 364
 365   // Update delay and max number of frames in flight based on the the new
 366   // received target delay.
 367   rtp_max_delay_ =
 368       base::TimeDelta::FromMilliseconds(cast_feedback.target_delay_ms_);
 369   max_unacked_frames_ = 1 + static_cast<uint8>(cast_feedback.target_delay_ms_ *
 370                                                max_frame_rate_ / 1000);
 371   if (rtcp_->Rtt(&rtt, &avg_rtt, &min_rtt, &max_rtt)) {
 372     // Don't use a RTT lower than our average.
 373     rtt = std::max(rtt, avg_rtt);
 374   } else {
 375     // We have no measured value use default.
 376     rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
 377   }
 378   if (cast_feedback.missing_frames_and_packets_.empty()) {
 379     // No lost packets.
 380     int resend_frame = -1;
 381     if (last_sent_frame_id_ == -1)
 382       return;
 383
 384     video_encoder_->LatestFrameIdToReference(cast_feedback.ack_frame_id_);
 385
 386     if (static_cast<uint32>(last_acked_frame_id_ + 1) ==
 387         cast_feedback.ack_frame_id_) {
 388       uint32 new_bitrate = 0;
 389       if (congestion_control_.OnAck(rtt, &new_bitrate)) {
 390         video_encoder_->SetBitRate(new_bitrate);
 391         current_requested_bitrate_ = new_bitrate;
 392       }
 393     }
 394     // We only count duplicate ACKs when we have sent newer frames.
 395     if (static_cast<uint32>(last_acked_frame_id_) ==
 396             cast_feedback.ack_frame_id_ &&
 397         IsNewerFrameId(last_sent_frame_id_, last_acked_frame_id_)) {
 398       duplicate_ack_++;
 399     } else {
 400       duplicate_ack_ = 0;
 401     }
 402     if (duplicate_ack_ >= 2 && duplicate_ack_ % 3 == 2) {
 403       // Resend last ACK + 1 frame.
 404       resend_frame = static_cast<uint32>(last_acked_frame_id_ + 1);
 405     }
 406     if (resend_frame != -1) {
 407       DCHECK_LE(0, resend_frame);
 408       VLOG(1) << "Received duplicate ACK for frame:"
 409               << static_cast<int>(resend_frame);
 410       ResendFrame(static_cast<uint32>(resend_frame));
 411     }
 412   } else {
 413     transport_sender_->ResendPackets(
 414         false, cast_feedback.missing_frames_and_packets_);
 415     uint32 new_bitrate = 0;
 416     if (congestion_control_.OnNack(rtt, &new_bitrate)) {
 417       video_encoder_->SetBitRate(new_bitrate);
 418       current_requested_bitrate_ = new_bitrate;
 419     }
 420   }
 421   ReceivedAck(cast_feedback.ack_frame_id_);
 422 }
 423
 424 void VideoSender::ReceivedAck(uint32 acked_frame_id) {
 425   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 426   if (acked_frame_id == UINT32_C(0xFFFFFFFF)) {
 427     // Receiver is sending a status message before any frames are ready to
 428     // be acked. Ignore.
 429     return;
 430   }
 431   // Start sending RTCP packets only after receiving the first ACK, i.e. only
 432   // after establishing that the receiver is active.
 433   if (last_acked_frame_id_ == -1) {
 434     ScheduleNextRtcpReport();
 435   }
 436   last_acked_frame_id_ = static_cast<int>(acked_frame_id);
 437   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 438
 439   RtpTimestamp rtp_timestamp =
 440       frame_id_to_rtp_timestamp_[acked_frame_id & 0xff];
 441   cast_environment_->Logging()->InsertFrameEvent(
 442       now, kVideoAckReceived, rtp_timestamp, acked_frame_id);
 443
 444   VLOG(2) << "ReceivedAck:" << static_cast<int>(acked_frame_id);
 445   active_session_ = true;
 446   DCHECK_NE(-1, last_acked_frame_id_);
 447   UpdateFramesInFlight();
 448 }
 449
 450 void VideoSender::UpdateFramesInFlight() {
 451   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 452   if (last_sent_frame_id_ != -1) {
 453     DCHECK_LE(0, last_sent_frame_id_);
 454     uint32 frames_in_flight = 0;
 455     if (last_acked_frame_id_ != -1) {
 456       DCHECK_LE(0, last_acked_frame_id_);
 457       frames_in_flight = static_cast<uint32>(last_sent_frame_id_) -
 458                          static_cast<uint32>(last_acked_frame_id_);
 459     } else {
 460       frames_in_flight = static_cast<uint32>(last_sent_frame_id_) + 1;
 461     }
 462     frames_in_flight += frames_in_encoder_;
 463     VLOG(2) << frames_in_flight
 464             << " Frames in flight; last sent: " << last_sent_frame_id_
 465             << " last acked:" << last_acked_frame_id_
 466             << " frames in encoder: " << frames_in_encoder_;
 467     if (frames_in_flight >= max_unacked_frames_) {
 468       video_encoder_->SkipNextFrame(true);
 469       return;
 470     }
 471     DCHECK(frames_in_flight <= max_unacked_frames_);
 472   }
 473   video_encoder_->SkipNextFrame(false);
 474 }
 475
 476 void VideoSender::ResendFrame(uint32 resend_frame_id) {
 477   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 478   MissingFramesAndPacketsMap missing_frames_and_packets;
 479   PacketIdSet missing;
 480   missing_frames_and_packets.insert(std::make_pair(resend_frame_id, missing));
 481   last_send_time_ = cast_environment_->Clock()->NowTicks();
 482   transport_sender_->ResendPackets(false, missing_frames_and_packets);
 483 }
 484
 485 }  // namespace cast
 486 }  // namespace media