media/cast/video_receiver/video_receiver.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_receiver/video_receiver.h"
   6
   7 #include <algorithm>
   8 #include "base/bind.h"
   9 #include "base/logging.h"
  10 #include "base/message_loop/message_loop.h"
  11 #include "media/cast/cast_defines.h"
  12 #include "media/cast/framer/framer.h"
  13 #include "media/cast/video_receiver/video_decoder.h"
  14
  15 namespace media {
  16 namespace cast {
  17
  18 const int64 kMinSchedulingDelayMs = 1;
  19
  20 static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000;
  21 static const int kTimeOffsetFilter = 8;
  22 static const int64_t kMinProcessIntervalMs = 5;
  23
  24 // Local implementation of RtpData (defined in rtp_rtcp_defines.h).
  25 // Used to pass payload data into the video receiver.
  26 class LocalRtpVideoData : public RtpData {
  27  public:
  28   explicit LocalRtpVideoData(base::TickClock* clock,
  29                              VideoReceiver* video_receiver)
  30       : clock_(clock),
  31         video_receiver_(video_receiver),
  32         time_updated_(false),
  33         incoming_rtp_timestamp_(0) {
  34   }
  35   virtual ~LocalRtpVideoData() {}
  36
  37   virtual void OnReceivedPayloadData(const uint8* payload_data,
  38                                      size_t payload_size,
  39                                      const RtpCastHeader* rtp_header) OVERRIDE {
  40     base::TimeTicks now = clock_->NowTicks();
  41     if (time_incoming_packet_.is_null() || now - time_incoming_packet_ >
  42         base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) {
  43       incoming_rtp_timestamp_ = rtp_header->webrtc.header.timestamp;
  44       time_incoming_packet_ = now;
  45       time_updated_ = true;
  46     }
  47     video_receiver_->IncomingRtpPacket(payload_data, payload_size, *rtp_header);
  48   }
  49
  50   bool GetPacketTimeInformation(base::TimeTicks* time_incoming_packet,
  51                                 uint32* incoming_rtp_timestamp) {
  52     *time_incoming_packet = time_incoming_packet_;
  53     *incoming_rtp_timestamp = incoming_rtp_timestamp_;
  54     bool time_updated = time_updated_;
  55     time_updated_ = false;
  56     return time_updated;
  57   }
  58
  59  private:
  60   base::TickClock* clock_;  // Not owned by this class.
  61   VideoReceiver* video_receiver_;
  62   bool time_updated_;
  63   base::TimeTicks time_incoming_packet_;
  64   uint32 incoming_rtp_timestamp_;
  65 };
  66
  67 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h)
  68 // Used to convey cast-specific feedback from receiver to sender.
  69 // Callback triggered by the Framer (cast message builder).
  70 class LocalRtpVideoFeedback : public RtpPayloadFeedback {
  71  public:
  72   explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver)
  73       : video_receiver_(video_receiver) {
  74   }
  75
  76   virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE {
  77     video_receiver_->CastFeedback(cast_message);
  78   }
  79
  80  private:
  81   VideoReceiver* video_receiver_;
  82 };
  83
  84 // Local implementation of RtpReceiverStatistics (defined by rtcp.h).
  85 // Used to pass statistics data from the RTP module to the RTCP module.
  86 class LocalRtpReceiverStatistics : public RtpReceiverStatistics {
  87  public:
  88   explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver)
  89      : rtp_receiver_(rtp_receiver) {
  90   }
  91
  92   virtual void GetStatistics(uint8* fraction_lost,
  93                              uint32* cumulative_lost,  // 24 bits valid.
  94                              uint32* extended_high_sequence_number,
  95                              uint32* jitter) OVERRIDE {
  96     rtp_receiver_->GetStatistics(fraction_lost,
  97                                  cumulative_lost,
  98                                  extended_high_sequence_number,
  99                                  jitter);
 100   }
 101
 102  private:
 103   RtpReceiver* rtp_receiver_;
 104 };
 105
 106 VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
 107                              const VideoReceiverConfig& video_config,
 108                              PacedPacketSender* const packet_sender)
 109       : cast_environment_(cast_environment),
 110         codec_(video_config.codec),
 111         incoming_ssrc_(video_config.incoming_ssrc),
 112         target_delay_delta_(
 113             base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
 114         frame_delay_(base::TimeDelta::FromMilliseconds(
 115             1000 / video_config.max_frame_rate)),
 116         incoming_payload_callback_(
 117             new LocalRtpVideoData(cast_environment_->Clock(), this)),
 118         incoming_payload_feedback_(new LocalRtpVideoFeedback(this)),
 119         rtp_receiver_(cast_environment_->Clock(), NULL, &video_config,
 120                       incoming_payload_callback_.get()),
 121         rtp_video_receiver_statistics_(
 122             new LocalRtpReceiverStatistics(&rtp_receiver_)),
 123         weak_factory_(this) {
 124   int max_unacked_frames = video_config.rtp_max_delay_ms *
 125       video_config.max_frame_rate / 1000;
 126   DCHECK(max_unacked_frames) << "Invalid argument";
 127
 128   framer_.reset(new Framer(cast_environment->Clock(),
 129                            incoming_payload_feedback_.get(),
 130                            video_config.incoming_ssrc,
 131                            video_config.decoder_faster_than_max_frame_rate,
 132                            max_unacked_frames));
 133   if (!video_config.use_external_decoder) {
 134     video_decoder_.reset(new VideoDecoder(video_config));
 135   }
 136
 137   rtcp_.reset(
 138       new Rtcp(cast_environment_->Clock(),
 139                NULL,
 140                packet_sender,
 141                NULL,
 142                rtp_video_receiver_statistics_.get(),
 143                video_config.rtcp_mode,
 144                base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
 145                false,
 146                video_config.feedback_ssrc,
 147                video_config.rtcp_c_name));
 148
 149   rtcp_->SetRemoteSSRC(video_config.incoming_ssrc);
 150   ScheduleNextRtcpReport();
 151   ScheduleNextCastMessage();
 152 }
 153
 154 VideoReceiver::~VideoReceiver() {}
 155
 156 void VideoReceiver::GetRawVideoFrame(
 157     const VideoFrameDecodedCallback& callback) {
 158   GetEncodedVideoFrame(base::Bind(&VideoReceiver::DecodeVideoFrame,
 159                                   weak_factory_.GetWeakPtr(),
 160                                   callback));
 161 }
 162
 163 // Called when we have a frame to decode.
 164 void VideoReceiver::DecodeVideoFrame(
 165     const VideoFrameDecodedCallback& callback,
 166     scoped_ptr<EncodedVideoFrame> encoded_frame,
 167     const base::TimeTicks& render_time) {
 168   // Hand the ownership of the encoded frame to the decode thread.
 169   cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER, FROM_HERE,
 170       base::Bind(&VideoReceiver::DecodeVideoFrameThread,
 171                  weak_factory_.GetWeakPtr(), base::Passed(&encoded_frame),
 172                  render_time, callback));
 173 }
 174
 175 // Utility function to run the decoder on a designated decoding thread.
 176 void VideoReceiver::DecodeVideoFrameThread(
 177     scoped_ptr<EncodedVideoFrame> encoded_frame,
 178     const base::TimeTicks render_time,
 179     const VideoFrameDecodedCallback& frame_decoded_callback) {
 180   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER));
 181   DCHECK(video_decoder_);
 182
 183   // TODO(mikhal): Allow the application to allocate this memory.
 184   scoped_ptr<I420VideoFrame> video_frame(new I420VideoFrame());
 185
 186   bool success = video_decoder_->DecodeVideoFrame(encoded_frame.get(),
 187       render_time, video_frame.get());
 188
 189   if (success) {
 190     VLOG(1) << "Decoded frame " << static_cast<int>(encoded_frame->frame_id);
 191     // Frame decoded - return frame to the user via callback.
 192     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 193           base::Bind(frame_decoded_callback,
 194                      base::Passed(&video_frame), render_time));
 195   } else {
 196     // This will happen if we decide to decode but not show a frame.
 197     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 198         base::Bind(&VideoReceiver::GetRawVideoFrame,
 199             weak_factory_.GetWeakPtr(), frame_decoded_callback));
 200   }
 201 }
 202
 203 // Called from the main cast thread.
 204 void VideoReceiver::GetEncodedVideoFrame(
 205     const VideoFrameEncodedCallback& callback) {
 206   scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame());
 207   uint32 rtp_timestamp = 0;
 208   bool next_frame = false;
 209
 210   if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
 211                                      &next_frame)) {
 212     // We have no video frames. Wait for new packet(s).
 213     queued_encoded_callbacks_.push_back(callback);
 214     return;
 215   }
 216   base::TimeTicks render_time;
 217   if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
 218                             &render_time)) {
 219     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 220         base::Bind(callback, base::Passed(&encoded_frame), render_time));
 221   } else {
 222     // We have a video frame; however we are missing packets and we have time
 223     // to wait for new packet(s).
 224     queued_encoded_callbacks_.push_back(callback);
 225   }
 226 }
 227
 228 // Should we pull the encoded video frame from the framer? decided by if this is
 229 // the next frame or we are running out of time and have to pull the following
 230 // frame.
 231 // If the frame it too old to be rendered we set the don't show flag in the
 232 // video bitstream where possible.
 233 bool VideoReceiver::PullEncodedVideoFrame(uint32 rtp_timestamp,
 234     bool next_frame, scoped_ptr<EncodedVideoFrame>* encoded_frame,
 235     base::TimeTicks* render_time) {
 236   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 237   *render_time = GetRenderTime(now, rtp_timestamp);
 238
 239   // Minimum time before a frame is due to be rendered before we pull it for
 240   // decode.
 241   base::TimeDelta min_wait_delta = frame_delay_;
 242   base::TimeDelta time_until_render = *render_time - now;
 243   if (!next_frame && (time_until_render > min_wait_delta)) {
 244     // Example:
 245     // We have decoded frame 1 and we have received the complete frame 3, but
 246     // not frame 2. If we still have time before frame 3 should be rendered we
 247     // will wait for 2 to arrive, however if 2 never show up this timer will hit
 248     // and we will pull out frame 3 for decoding and rendering.
 249     base::TimeDelta time_until_release = time_until_render - min_wait_delta;
 250     cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 251         base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()),
 252         time_until_release);
 253     VLOG(0) << "Wait before releasing frame "
 254             << static_cast<int>((*encoded_frame)->frame_id)
 255             << " time " << time_until_release.InMilliseconds();
 256     return false;
 257   }
 258
 259   base::TimeDelta dont_show_timeout_delta =
 260       base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs);
 261   if (codec_ == kVp8 && time_until_render < dont_show_timeout_delta) {
 262     (*encoded_frame)->data[0] &= 0xef;
 263     VLOG(0) << "Don't show frame "
 264             << static_cast<int>((*encoded_frame)->frame_id)
 265             << " time_until_render:" << time_until_render.InMilliseconds();
 266   } else {
 267     VLOG(1) << "Show frame "
 268             << static_cast<int>((*encoded_frame)->frame_id)
 269             << " time_until_render:" << time_until_render.InMilliseconds();
 270   }
 271   // We have a copy of the frame, release this one.
 272   framer_->ReleaseFrame((*encoded_frame)->frame_id);
 273   (*encoded_frame)->codec = codec_;
 274   return true;
 275 }
 276
 277 void VideoReceiver::PlayoutTimeout() {
 278   if (queued_encoded_callbacks_.empty()) return;
 279
 280   uint32 rtp_timestamp = 0;
 281   bool next_frame = false;
 282   scoped_ptr<EncodedVideoFrame> encoded_frame(new EncodedVideoFrame());
 283
 284   if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
 285                                      &next_frame)) {
 286     // We have no video frames. Wait for new packet(s).
 287     // Since the application can post multiple VideoFrameEncodedCallback and
 288     // we only check the next frame to play out we might have multiple timeout
 289     // events firing after each other; however this should be a rare event.
 290     VLOG(1) << "Failed to retrieved a complete frame at this point in time";
 291     return;
 292   }
 293   VLOG(1) << "PlayoutTimeout retrieved frame "
 294           << static_cast<int>(encoded_frame->frame_id);
 295
 296   base::TimeTicks render_time;
 297   if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
 298                             &render_time)) {
 299     if (!queued_encoded_callbacks_.empty()) {
 300       VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 301       queued_encoded_callbacks_.pop_front();
 302       cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 303           base::Bind(callback, base::Passed(&encoded_frame), render_time));
 304     }
 305   } else {
 306     // We have a video frame; however we are missing packets and we have time
 307     // to wait for new packet(s).
 308   }
 309 }
 310
 311 base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now,
 312                                              uint32 rtp_timestamp) {
 313   // Senders time in ms when this frame was captured.
 314   // Note: the senders clock and our local clock might not be synced.
 315   base::TimeTicks rtp_timestamp_in_ticks;
 316   base::TimeTicks time_incoming_packet;
 317   uint32 incoming_rtp_timestamp;
 318
 319   if (time_offset_.InMilliseconds() == 0) {
 320     incoming_payload_callback_->GetPacketTimeInformation(
 321         &time_incoming_packet, &incoming_rtp_timestamp);
 322
 323     if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 324                                          incoming_rtp_timestamp,
 325                                          &rtp_timestamp_in_ticks)) {
 326       // We have not received any RTCP to sync the stream play it out as soon as
 327       // possible.
 328       return now;
 329     }
 330     time_offset_ = time_incoming_packet - rtp_timestamp_in_ticks;
 331   } else if (incoming_payload_callback_->GetPacketTimeInformation(
 332       &time_incoming_packet, &incoming_rtp_timestamp)) {
 333     if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 334                                         incoming_rtp_timestamp,
 335                                         &rtp_timestamp_in_ticks)) {
 336       // Time to update the time_offset.
 337       base::TimeDelta time_offset =
 338           time_incoming_packet - rtp_timestamp_in_ticks;
 339       time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset)
 340           / kTimeOffsetFilter;
 341     }
 342   }
 343   if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 344                                        rtp_timestamp,
 345                                        &rtp_timestamp_in_ticks)) {
 346     // This can fail if we have not received any RTCP packets in a long time.
 347     return now;
 348   }
 349   return (rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_);
 350 }
 351
 352 void VideoReceiver::IncomingPacket(const uint8* packet, size_t length,
 353                                    const base::Closure callback) {
 354   if (Rtcp::IsRtcpPacket(packet, length)) {
 355     rtcp_->IncomingRtcpPacket(packet, length);
 356   } else {
 357     rtp_receiver_.ReceivedPacket(packet, length);
 358   }
 359   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
 360 }
 361
 362 void VideoReceiver::IncomingRtpPacket(const uint8* payload_data,
 363                                       size_t payload_size,
 364                                       const RtpCastHeader& rtp_header) {
 365   bool complete = framer_->InsertPacket(payload_data, payload_size, rtp_header);
 366
 367   if (!complete) return;  // Video frame not complete; wait for more packets.
 368   if (queued_encoded_callbacks_.empty()) return;  // No pending callback.
 369
 370   VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 371   queued_encoded_callbacks_.pop_front();
 372   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 373       base::Bind(&VideoReceiver::GetEncodedVideoFrame,
 374           weak_factory_.GetWeakPtr(), callback));
 375 }
 376
 377 // Send a cast feedback message. Actual message created in the framer (cast
 378 // message builder).
 379 void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
 380   rtcp_->SendRtcpCast(cast_message);
 381   time_last_sent_cast_message_= cast_environment_->Clock()->NowTicks();
 382 }
 383
 384 // Send a key frame request to the sender.
 385 void VideoReceiver::RequestKeyFrame() {
 386   rtcp_->SendRtcpPli(incoming_ssrc_);
 387 }
 388
 389 // Cast messages should be sent within a maximum interval. Schedule a call
 390 // if not triggered elsewhere, e.g. by the cast message_builder.
 391 void VideoReceiver::ScheduleNextCastMessage() {
 392   base::TimeTicks send_time;
 393   framer_->TimeToSendNextCastMessage(&send_time);
 394
 395   base::TimeDelta time_to_send = send_time -
 396       cast_environment_->Clock()->NowTicks();
 397   time_to_send = std::max(time_to_send,
 398       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 399   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 400       base::Bind(&VideoReceiver::SendNextCastMessage,
 401                  weak_factory_.GetWeakPtr()), time_to_send);
 402 }
 403
 404 void VideoReceiver::SendNextCastMessage() {
 405   framer_->SendCastMessage();  // Will only send a message if it is time.
 406   ScheduleNextCastMessage();
 407 }
 408
 409 // Schedule the next RTCP report to be sent back to the sender.
 410 void VideoReceiver::ScheduleNextRtcpReport() {
 411   base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
 412       cast_environment_->Clock()->NowTicks();
 413
 414   time_to_next = std::max(time_to_next,
 415       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 416
 417   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 418       base::Bind(&VideoReceiver::SendNextRtcpReport,
 419                 weak_factory_.GetWeakPtr()), time_to_next);
 420 }
 421
 422 void VideoReceiver::SendNextRtcpReport() {
 423   rtcp_->SendRtcpReport(incoming_ssrc_);
 424   ScheduleNextRtcpReport();
 425 }
 426
 427 }  // namespace cast
 428 }  // namespace media