media/cast/video_receiver/video_receiver.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_receiver/video_receiver.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "media/cast/cast_defines.h"
  13 #include "media/cast/framer/framer.h"
  14 #include "media/cast/rtcp/receiver_rtcp_event_subscriber.h"
  15 #include "media/cast/rtcp/rtcp_sender.h"
  16 #include "media/cast/video_receiver/video_decoder.h"
  17
  18 namespace {
  19
  20 using media::cast::kMaxIpPacketSize;
  21 using media::cast::kRtcpCastLogHeaderSize;
  22 using media::cast::kRtcpReceiverEventLogSize;
  23
  24 static const int64 kMinSchedulingDelayMs = 1;
  25
  26 static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000;
  27 static const int kTimeOffsetFilter = 8;
  28 static const int64_t kMinProcessIntervalMs = 5;
  29
  30 // This is an upper bound on number of events that can fit into a single RTCP
  31 // packet.
  32 static const int64 kMaxEventSubscriberEntries =
  33     (kMaxIpPacketSize - kRtcpCastLogHeaderSize) / kRtcpReceiverEventLogSize;
  34
  35 }  // namespace
  36
  37 namespace media {
  38 namespace cast {
  39
  40 // Local implementation of RtpData (defined in rtp_rtcp_defines.h).
  41 // Used to pass payload data into the video receiver.
  42 class LocalRtpVideoData : public RtpData {
  43  public:
  44   explicit LocalRtpVideoData(VideoReceiver* video_receiver)
  45       : video_receiver_(video_receiver) {}
  46
  47   virtual ~LocalRtpVideoData() {}
  48
  49   virtual void OnReceivedPayloadData(const uint8* payload_data,
  50                                      size_t payload_size,
  51                                      const RtpCastHeader* rtp_header) OVERRIDE {
  52     video_receiver_->IncomingParsedRtpPacket(
  53         payload_data, payload_size, *rtp_header);
  54   }
  55
  56  private:
  57   VideoReceiver* video_receiver_;
  58
  59   DISALLOW_IMPLICIT_CONSTRUCTORS(LocalRtpVideoData);
  60 };
  61
  62 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h)
  63 // Used to convey cast-specific feedback from receiver to sender.
  64 // Callback triggered by the Framer (cast message builder).
  65 class LocalRtpVideoFeedback : public RtpPayloadFeedback {
  66  public:
  67   explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver)
  68       : video_receiver_(video_receiver) {}
  69
  70   virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE {
  71     video_receiver_->CastFeedback(cast_message);
  72   }
  73
  74  private:
  75   VideoReceiver* video_receiver_;
  76
  77   DISALLOW_IMPLICIT_CONSTRUCTORS(LocalRtpVideoFeedback);
  78 };
  79
  80 // Local implementation of RtpReceiverStatistics (defined by rtcp.h).
  81 // Used to pass statistics data from the RTP module to the RTCP module.
  82 class LocalRtpReceiverStatistics : public RtpReceiverStatistics {
  83  public:
  84   explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver)
  85       : rtp_receiver_(rtp_receiver) {}
  86
  87   virtual void GetStatistics(uint8* fraction_lost,
  88                              uint32* cumulative_lost,  // 24 bits valid.
  89                              uint32* extended_high_sequence_number,
  90                              uint32* jitter) OVERRIDE {
  91     rtp_receiver_->GetStatistics(
  92         fraction_lost, cumulative_lost, extended_high_sequence_number, jitter);
  93   }
  94
  95  private:
  96   RtpReceiver* rtp_receiver_;
  97
  98   DISALLOW_IMPLICIT_CONSTRUCTORS(LocalRtpReceiverStatistics);
  99 };
 100
 101 VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
 102                              const VideoReceiverConfig& video_config,
 103                              transport::PacedPacketSender* const packet_sender)
 104     : cast_environment_(cast_environment),
 105       event_subscriber_(
 106           kMaxEventSubscriberEntries,
 107           ReceiverRtcpEventSubscriber::kVideoEventSubscriber),
 108       codec_(video_config.codec),
 109       target_delay_delta_(
 110           base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
 111       frame_delay_(base::TimeDelta::FromMilliseconds(
 112           1000 / video_config.max_frame_rate)),
 113       incoming_payload_callback_(new LocalRtpVideoData(this)),
 114       incoming_payload_feedback_(new LocalRtpVideoFeedback(this)),
 115       rtp_receiver_(cast_environment_->Clock(),
 116                     NULL,
 117                     &video_config,
 118                     incoming_payload_callback_.get()),
 119       rtp_video_receiver_statistics_(
 120           new LocalRtpReceiverStatistics(&rtp_receiver_)),
 121       decryptor_(),
 122       time_incoming_packet_updated_(false),
 123       incoming_rtp_timestamp_(0),
 124       weak_factory_(this) {
 125   int max_unacked_frames =
 126       video_config.rtp_max_delay_ms * video_config.max_frame_rate / 1000;
 127   DCHECK(max_unacked_frames) << "Invalid argument";
 128
 129   decryptor_.Initialize(video_config.aes_key, video_config.aes_iv_mask);
 130   framer_.reset(new Framer(cast_environment->Clock(),
 131                            incoming_payload_feedback_.get(),
 132                            video_config.incoming_ssrc,
 133                            video_config.decoder_faster_than_max_frame_rate,
 134                            max_unacked_frames));
 135
 136   if (!video_config.use_external_decoder) {
 137     video_decoder_.reset(new VideoDecoder(video_config, cast_environment));
 138   }
 139
 140   rtcp_.reset(
 141       new Rtcp(cast_environment_,
 142                NULL,
 143                NULL,
 144                packet_sender,
 145                NULL,
 146                rtp_video_receiver_statistics_.get(),
 147                video_config.rtcp_mode,
 148                base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
 149                video_config.feedback_ssrc,
 150                video_config.incoming_ssrc,
 151                video_config.rtcp_c_name));
 152   cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_);
 153 }
 154
 155 VideoReceiver::~VideoReceiver() {
 156   cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_);
 157 }
 158
 159 void VideoReceiver::InitializeTimers() {
 160   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 161   ScheduleNextRtcpReport();
 162   ScheduleNextCastMessage();
 163 }
 164
 165 void VideoReceiver::GetRawVideoFrame(
 166     const VideoFrameDecodedCallback& callback) {
 167   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 168   GetEncodedVideoFrame(base::Bind(
 169       &VideoReceiver::DecodeVideoFrame, base::Unretained(this), callback));
 170 }
 171
 172 // Called when we have a frame to decode.
 173 void VideoReceiver::DecodeVideoFrame(
 174     const VideoFrameDecodedCallback& callback,
 175     scoped_ptr<transport::EncodedVideoFrame> encoded_frame,
 176     const base::TimeTicks& render_time) {
 177   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 178   // Hand the ownership of the encoded frame to the decode thread.
 179   cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER,
 180                               FROM_HERE,
 181                               base::Bind(&VideoReceiver::DecodeVideoFrameThread,
 182                                          base::Unretained(this),
 183                                          base::Passed(&encoded_frame),
 184                                          render_time,
 185                                          callback));
 186 }
 187
 188 // Utility function to run the decoder on a designated decoding thread.
 189 void VideoReceiver::DecodeVideoFrameThread(
 190     scoped_ptr<transport::EncodedVideoFrame> encoded_frame,
 191     const base::TimeTicks render_time,
 192     const VideoFrameDecodedCallback& frame_decoded_callback) {
 193   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER));
 194   DCHECK(video_decoder_);
 195
 196   if (!(video_decoder_->DecodeVideoFrame(
 197            encoded_frame.get(), render_time, frame_decoded_callback))) {
 198     // This will happen if we decide to decode but not show a frame.
 199     cast_environment_->PostTask(CastEnvironment::MAIN,
 200                                 FROM_HERE,
 201                                 base::Bind(&VideoReceiver::GetRawVideoFrame,
 202                                            base::Unretained(this),
 203                                            frame_decoded_callback));
 204   }
 205 }
 206
 207 bool VideoReceiver::DecryptVideoFrame(
 208     scoped_ptr<transport::EncodedVideoFrame>* video_frame) {
 209   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 210
 211   if (!decryptor_.initialized())
 212     return false;
 213
 214   std::string decrypted_video_data;
 215   if (!decryptor_.Decrypt((*video_frame)->frame_id,
 216                           (*video_frame)->data,
 217                           &decrypted_video_data)) {
 218     // Give up on this frame, release it from jitter buffer.
 219     framer_->ReleaseFrame((*video_frame)->frame_id);
 220     return false;
 221   }
 222   (*video_frame)->data.swap(decrypted_video_data);
 223   return true;
 224 }
 225
 226 // Called from the main cast thread.
 227 void VideoReceiver::GetEncodedVideoFrame(
 228     const VideoFrameEncodedCallback& callback) {
 229   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 230   scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
 231       new transport::EncodedVideoFrame());
 232   uint32 rtp_timestamp = 0;
 233   bool next_frame = false;
 234
 235   if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &next_frame)) {
 236     // We have no video frames. Wait for new packet(s).
 237     queued_encoded_callbacks_.push_back(callback);
 238     return;
 239   }
 240
 241   if (decryptor_.initialized() && !DecryptVideoFrame(&encoded_frame)) {
 242     // Logging already done.
 243     queued_encoded_callbacks_.push_back(callback);
 244     return;
 245   }
 246
 247   base::TimeTicks render_time;
 248   if (PullEncodedVideoFrame(next_frame, &encoded_frame, &render_time)) {
 249     cast_environment_->PostTask(
 250         CastEnvironment::MAIN,
 251         FROM_HERE,
 252         base::Bind(callback, base::Passed(&encoded_frame), render_time));
 253   } else {
 254     // We have a video frame; however we are missing packets and we have time
 255     // to wait for new packet(s).
 256     queued_encoded_callbacks_.push_back(callback);
 257   }
 258 }
 259
 260 // Should we pull the encoded video frame from the framer? decided by if this is
 261 // the next frame or we are running out of time and have to pull the following
 262 // frame.
 263 // If the frame is too old to be rendered we set the don't show flag in the
 264 // video bitstream where possible.
 265 bool VideoReceiver::PullEncodedVideoFrame(
 266     bool next_frame,
 267     scoped_ptr<transport::EncodedVideoFrame>* encoded_frame,
 268     base::TimeTicks* render_time) {
 269   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 270   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 271   *render_time = GetRenderTime(now, (*encoded_frame)->rtp_timestamp);
 272
 273   // TODO(mikhal): Store actual render time and not diff.
 274   cast_environment_->Logging()->InsertFrameEventWithDelay(
 275       now,
 276       kVideoRenderDelay,
 277       (*encoded_frame)->rtp_timestamp,
 278       (*encoded_frame)->frame_id,
 279       now - *render_time);
 280
 281   // Minimum time before a frame is due to be rendered before we pull it for
 282   // decode.
 283   base::TimeDelta min_wait_delta = frame_delay_;
 284   base::TimeDelta time_until_render = *render_time - now;
 285   if (!next_frame && (time_until_render > min_wait_delta)) {
 286     // Example:
 287     // We have decoded frame 1 and we have received the complete frame 3, but
 288     // not frame 2. If we still have time before frame 3 should be rendered we
 289     // will wait for 2 to arrive, however if 2 never show up this timer will hit
 290     // and we will pull out frame 3 for decoding and rendering.
 291     base::TimeDelta time_until_release = time_until_render - min_wait_delta;
 292     cast_environment_->PostDelayedTask(
 293         CastEnvironment::MAIN,
 294         FROM_HERE,
 295         base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()),
 296         time_until_release);
 297     VLOG(1) << "Wait before releasing frame "
 298             << static_cast<int>((*encoded_frame)->frame_id) << " time "
 299             << time_until_release.InMilliseconds();
 300     return false;
 301   }
 302
 303   base::TimeDelta dont_show_timeout_delta =
 304       base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs);
 305   if (codec_ == transport::kVp8 &&
 306       time_until_render < dont_show_timeout_delta) {
 307     (*encoded_frame)->data[0] &= 0xef;
 308     VLOG(1) << "Don't show frame "
 309             << static_cast<int>((*encoded_frame)->frame_id)
 310             << " time_until_render:" << time_until_render.InMilliseconds();
 311   } else {
 312     VLOG(1) << "Show frame " << static_cast<int>((*encoded_frame)->frame_id)
 313             << " time_until_render:" << time_until_render.InMilliseconds();
 314   }
 315   // We have a copy of the frame, release this one.
 316   framer_->ReleaseFrame((*encoded_frame)->frame_id);
 317   (*encoded_frame)->codec = codec_;
 318   return true;
 319 }
 320
 321 void VideoReceiver::PlayoutTimeout() {
 322   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 323   if (queued_encoded_callbacks_.empty())
 324     return;
 325
 326   bool next_frame = false;
 327   scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
 328       new transport::EncodedVideoFrame());
 329
 330   if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &next_frame)) {
 331     // We have no video frames. Wait for new packet(s).
 332     // Since the application can post multiple VideoFrameEncodedCallback and
 333     // we only check the next frame to play out we might have multiple timeout
 334     // events firing after each other; however this should be a rare event.
 335     VLOG(1) << "Failed to retrieved a complete frame at this point in time";
 336     return;
 337   }
 338   VLOG(1) << "PlayoutTimeout retrieved frame "
 339           << static_cast<int>(encoded_frame->frame_id);
 340
 341   if (decryptor_.initialized() && !DecryptVideoFrame(&encoded_frame)) {
 342     // Logging already done.
 343     return;
 344   }
 345
 346   base::TimeTicks render_time;
 347   if (PullEncodedVideoFrame(next_frame, &encoded_frame, &render_time)) {
 348     if (!queued_encoded_callbacks_.empty()) {
 349       VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 350       queued_encoded_callbacks_.pop_front();
 351       cast_environment_->PostTask(
 352           CastEnvironment::MAIN,
 353           FROM_HERE,
 354           base::Bind(callback, base::Passed(&encoded_frame), render_time));
 355     }
 356   }
 357   // Else we have a video frame; however we are missing packets and we have time
 358   // to wait for new packet(s).
 359 }
 360
 361 base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now,
 362                                              uint32 rtp_timestamp) {
 363   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 364   // Senders time in ms when this frame was captured.
 365   // Note: the senders clock and our local clock might not be synced.
 366   base::TimeTicks rtp_timestamp_in_ticks;
 367
 368   // Compute the time offset_in_ticks based on the incoming_rtp_timestamp_.
 369   if (time_offset_.InMilliseconds() == 0) {
 370     if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 371                                          incoming_rtp_timestamp_,
 372                                          &rtp_timestamp_in_ticks)) {
 373       // We have not received any RTCP to sync the stream play it out as soon as
 374       // possible.
 375       return now;
 376     }
 377     time_offset_ = time_incoming_packet_ - rtp_timestamp_in_ticks;
 378   } else if (time_incoming_packet_updated_) {
 379     if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 380                                         incoming_rtp_timestamp_,
 381                                         &rtp_timestamp_in_ticks)) {
 382       // Time to update the time_offset.
 383       base::TimeDelta time_offset =
 384           time_incoming_packet_ - rtp_timestamp_in_ticks;
 385       time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset) /
 386                      kTimeOffsetFilter;
 387     }
 388   }
 389   // Reset |time_incoming_packet_updated_| to enable a future measurement.
 390   time_incoming_packet_updated_ = false;
 391   // Compute the actual rtp_timestamp_in_ticks based on the current timestamp.
 392   if (!rtcp_->RtpTimestampInSenderTime(
 393            kVideoFrequency, rtp_timestamp, &rtp_timestamp_in_ticks)) {
 394     // This can fail if we have not received any RTCP packets in a long time.
 395     return now;
 396   }
 397   base::TimeTicks render_time =
 398       rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_;
 399   if (last_render_time_ > render_time)
 400     render_time = last_render_time_;
 401   last_render_time_ = render_time;
 402   return render_time;
 403 }
 404
 405 void VideoReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
 406   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 407   if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) {
 408     rtcp_->IncomingRtcpPacket(&packet->front(), packet->size());
 409   } else {
 410     rtp_receiver_.ReceivedPacket(&packet->front(), packet->size());
 411   }
 412 }
 413
 414 void VideoReceiver::IncomingParsedRtpPacket(const uint8* payload_data,
 415                                             size_t payload_size,
 416                                             const RtpCastHeader& rtp_header) {
 417   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 418
 419   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 420   if (time_incoming_packet_.is_null() ||
 421       now - time_incoming_packet_ >
 422           base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) {
 423     if (time_incoming_packet_.is_null())
 424       InitializeTimers();
 425     incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp;
 426     time_incoming_packet_ = now;
 427     time_incoming_packet_updated_ = true;
 428   }
 429
 430   cast_environment_->Logging()->InsertPacketEvent(
 431       now,
 432       kVideoPacketReceived,
 433       rtp_header.webrtc.header.timestamp,
 434       rtp_header.frame_id,
 435       rtp_header.packet_id,
 436       rtp_header.max_packet_id,
 437       payload_size);
 438
 439   bool duplicate = false;
 440   bool complete =
 441       framer_->InsertPacket(payload_data, payload_size, rtp_header, &duplicate);
 442
 443   if (duplicate) {
 444     cast_environment_->Logging()->InsertPacketEvent(
 445         now,
 446         kDuplicatePacketReceived,
 447         rtp_header.webrtc.header.timestamp,
 448         rtp_header.frame_id,
 449         rtp_header.packet_id,
 450         rtp_header.max_packet_id,
 451         payload_size);
 452     // Duplicate packets are ignored.
 453     return;
 454   }
 455   if (!complete)
 456     return;  // Video frame not complete; wait for more packets.
 457   if (queued_encoded_callbacks_.empty())
 458     return;  // No pending callback.
 459
 460   VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 461   queued_encoded_callbacks_.pop_front();
 462   cast_environment_->PostTask(CastEnvironment::MAIN,
 463                               FROM_HERE,
 464                               base::Bind(&VideoReceiver::GetEncodedVideoFrame,
 465                                          weak_factory_.GetWeakPtr(),
 466                                          callback));
 467 }
 468
 469 // Send a cast feedback message. Actual message created in the framer (cast
 470 // message builder).
 471 void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
 472   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 473
 474   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 475   cast_environment_->Logging()->InsertGenericEvent(
 476       now, kVideoAckSent, cast_message.ack_frame_id_);
 477
 478   rtcp_->SendRtcpFromRtpReceiver(&cast_message, &event_subscriber_);
 479 }
 480
 481 // Cast messages should be sent within a maximum interval. Schedule a call
 482 // if not triggered elsewhere, e.g. by the cast message_builder.
 483 void VideoReceiver::ScheduleNextCastMessage() {
 484   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 485   base::TimeTicks send_time;
 486   framer_->TimeToSendNextCastMessage(&send_time);
 487
 488   base::TimeDelta time_to_send =
 489       send_time - cast_environment_->Clock()->NowTicks();
 490   time_to_send = std::max(
 491       time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 492   cast_environment_->PostDelayedTask(
 493       CastEnvironment::MAIN,
 494       FROM_HERE,
 495       base::Bind(&VideoReceiver::SendNextCastMessage,
 496                  weak_factory_.GetWeakPtr()),
 497       time_to_send);
 498 }
 499
 500 void VideoReceiver::SendNextCastMessage() {
 501   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 502   framer_->SendCastMessage();  // Will only send a message if it is time.
 503   ScheduleNextCastMessage();
 504 }
 505
 506 // Schedule the next RTCP report to be sent back to the sender.
 507 void VideoReceiver::ScheduleNextRtcpReport() {
 508   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 509   base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
 510                                  cast_environment_->Clock()->NowTicks();
 511
 512   time_to_next = std::max(
 513       time_to_next, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 514
 515   cast_environment_->PostDelayedTask(
 516       CastEnvironment::MAIN,
 517       FROM_HERE,
 518       base::Bind(&VideoReceiver::SendNextRtcpReport,
 519                  weak_factory_.GetWeakPtr()),
 520       time_to_next);
 521 }
 522
 523 void VideoReceiver::SendNextRtcpReport() {
 524   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 525   rtcp_->SendRtcpFromRtpReceiver(NULL, NULL);
 526   ScheduleNextRtcpReport();
 527 }
 528
 529 }  // namespace cast
 530 }  // namespace media