media/cast/video_receiver/video_receiver.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/video_receiver/video_receiver.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "crypto/encryptor.h"
  13 #include "crypto/symmetric_key.h"
  14 #include "media/cast/cast_defines.h"
  15 #include "media/cast/framer/framer.h"
  16 #include "media/cast/video_receiver/video_decoder.h"
  17
  18 namespace media {
  19 namespace cast {
  20
  21 const int64 kMinSchedulingDelayMs = 1;
  22
  23 static const int64 kMinTimeBetweenOffsetUpdatesMs = 2000;
  24 static const int kTimeOffsetFilter = 8;
  25 static const int64_t kMinProcessIntervalMs = 5;
  26
  27 // Local implementation of RtpData (defined in rtp_rtcp_defines.h).
  28 // Used to pass payload data into the video receiver.
  29 class LocalRtpVideoData : public RtpData {
  30  public:
  31   explicit LocalRtpVideoData(VideoReceiver* video_receiver)
  32       : video_receiver_(video_receiver) {}
  33
  34   virtual ~LocalRtpVideoData() {}
  35
  36   virtual void OnReceivedPayloadData(const uint8* payload_data,
  37                                      size_t payload_size,
  38                                      const RtpCastHeader* rtp_header) OVERRIDE {
  39     video_receiver_->IncomingParsedRtpPacket(payload_data, payload_size,
  40                                              *rtp_header);
  41   }
  42
  43  private:
  44   VideoReceiver* video_receiver_;
  45 };
  46
  47 // Local implementation of RtpPayloadFeedback (defined in rtp_defines.h)
  48 // Used to convey cast-specific feedback from receiver to sender.
  49 // Callback triggered by the Framer (cast message builder).
  50 class LocalRtpVideoFeedback : public RtpPayloadFeedback {
  51  public:
  52   explicit LocalRtpVideoFeedback(VideoReceiver* video_receiver)
  53       : video_receiver_(video_receiver) {
  54   }
  55
  56   virtual void CastFeedback(const RtcpCastMessage& cast_message) OVERRIDE {
  57     video_receiver_->CastFeedback(cast_message);
  58   }
  59
  60  private:
  61   VideoReceiver* video_receiver_;
  62 };
  63
  64 // Local implementation of RtpReceiverStatistics (defined by rtcp.h).
  65 // Used to pass statistics data from the RTP module to the RTCP module.
  66 class LocalRtpReceiverStatistics : public RtpReceiverStatistics {
  67  public:
  68   explicit LocalRtpReceiverStatistics(RtpReceiver* rtp_receiver)
  69      : rtp_receiver_(rtp_receiver) {
  70   }
  71
  72   virtual void GetStatistics(uint8* fraction_lost,
  73                              uint32* cumulative_lost,  // 24 bits valid.
  74                              uint32* extended_high_sequence_number,
  75                              uint32* jitter) OVERRIDE {
  76     rtp_receiver_->GetStatistics(fraction_lost,
  77                                  cumulative_lost,
  78                                  extended_high_sequence_number,
  79                                  jitter);
  80   }
  81
  82  private:
  83   RtpReceiver* rtp_receiver_;
  84 };
  85
  86 VideoReceiver::VideoReceiver(scoped_refptr<CastEnvironment> cast_environment,
  87                              const VideoReceiverConfig& video_config,
  88                              transport::PacedPacketSender* const packet_sender)
  89       : cast_environment_(cast_environment),
  90         codec_(video_config.codec),
  91         target_delay_delta_(
  92             base::TimeDelta::FromMilliseconds(video_config.rtp_max_delay_ms)),
  93         frame_delay_(base::TimeDelta::FromMilliseconds(
  94             1000 / video_config.max_frame_rate)),
  95         incoming_payload_callback_(new LocalRtpVideoData(this)),
  96         incoming_payload_feedback_(new LocalRtpVideoFeedback(this)),
  97         rtp_receiver_(cast_environment_->Clock(), NULL, &video_config,
  98                       incoming_payload_callback_.get()),
  99         rtp_video_receiver_statistics_(
 100             new LocalRtpReceiverStatistics(&rtp_receiver_)),
 101         time_incoming_packet_updated_(false),
 102         incoming_rtp_timestamp_(0),
 103         weak_factory_(this) {
 104   int max_unacked_frames = video_config.rtp_max_delay_ms *
 105       video_config.max_frame_rate / 1000;
 106   DCHECK(max_unacked_frames) << "Invalid argument";
 107
 108   if (video_config.aes_iv_mask.size() == kAesKeySize &&
 109       video_config.aes_key.size() == kAesKeySize) {
 110     iv_mask_ = video_config.aes_iv_mask;
 111     decryption_key_.reset(crypto::SymmetricKey::Import(
 112         crypto::SymmetricKey::AES, video_config.aes_key));
 113     decryptor_.reset(new crypto::Encryptor());
 114     decryptor_->Init(decryption_key_.get(),
 115                      crypto::Encryptor::CTR,
 116                      std::string());
 117   } else if (video_config.aes_iv_mask.size() != 0 ||
 118              video_config.aes_key.size() != 0) {
 119     DCHECK(false) << "Invalid crypto configuration";
 120   }
 121
 122   framer_.reset(new Framer(cast_environment->Clock(),
 123                            incoming_payload_feedback_.get(),
 124                            video_config.incoming_ssrc,
 125                            video_config.decoder_faster_than_max_frame_rate,
 126                            max_unacked_frames));
 127   if (!video_config.use_external_decoder) {
 128     video_decoder_.reset(new VideoDecoder(video_config, cast_environment));
 129   }
 130
 131   rtcp_.reset(
 132       new Rtcp(cast_environment_,
 133                NULL,
 134                packet_sender,
 135                NULL,
 136                rtp_video_receiver_statistics_.get(),
 137                video_config.rtcp_mode,
 138                base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
 139                video_config.feedback_ssrc,
 140                video_config.incoming_ssrc,
 141                video_config.rtcp_c_name));
 142 }
 143
 144 VideoReceiver::~VideoReceiver() {}
 145
 146 void VideoReceiver::InitializeTimers() {
 147   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 148   ScheduleNextRtcpReport();
 149   ScheduleNextCastMessage();
 150 }
 151
 152 void VideoReceiver::GetRawVideoFrame(
 153     const VideoFrameDecodedCallback& callback) {
 154   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 155   GetEncodedVideoFrame(base::Bind(&VideoReceiver::DecodeVideoFrame,
 156                                   base::Unretained(this), callback));
 157 }
 158
 159 // Called when we have a frame to decode.
 160 void VideoReceiver::DecodeVideoFrame(
 161     const VideoFrameDecodedCallback& callback,
 162     scoped_ptr<transport::EncodedVideoFrame> encoded_frame,
 163     const base::TimeTicks& render_time) {
 164   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 165   // Hand the ownership of the encoded frame to the decode thread.
 166   cast_environment_->PostTask(CastEnvironment::VIDEO_DECODER, FROM_HERE,
 167       base::Bind(&VideoReceiver::DecodeVideoFrameThread, base::Unretained(this),
 168                  base::Passed(&encoded_frame), render_time, callback));
 169 }
 170
 171 // Utility function to run the decoder on a designated decoding thread.
 172 void VideoReceiver::DecodeVideoFrameThread(
 173     scoped_ptr<transport::EncodedVideoFrame> encoded_frame,
 174     const base::TimeTicks render_time,
 175     const VideoFrameDecodedCallback& frame_decoded_callback) {
 176   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::VIDEO_DECODER));
 177   DCHECK(video_decoder_);
 178
 179   if (!(video_decoder_->DecodeVideoFrame(encoded_frame.get(), render_time,
 180         frame_decoded_callback))) {
 181     // This will happen if we decide to decode but not show a frame.
 182     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 183         base::Bind(&VideoReceiver::GetRawVideoFrame, base::Unretained(this),
 184                    frame_decoded_callback));
 185   }
 186 }
 187
 188 bool VideoReceiver::DecryptVideoFrame(
 189     scoped_ptr<transport::EncodedVideoFrame>* video_frame) {
 190   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 191   DCHECK(decryptor_) << "Invalid state";
 192
 193   if (!decryptor_->SetCounter(GetAesNonce((*video_frame)->frame_id,
 194                                           iv_mask_))) {
 195     NOTREACHED() << "Failed to set counter";
 196     return false;
 197   }
 198   std::string decrypted_video_data;
 199   if (!decryptor_->Decrypt((*video_frame)->data, &decrypted_video_data)) {
 200     VLOG(1) << "Decryption error";
 201     // Give up on this frame, release it from jitter buffer.
 202     framer_->ReleaseFrame((*video_frame)->frame_id);
 203     return false;
 204   }
 205   (*video_frame)->data.swap(decrypted_video_data);
 206   return true;
 207 }
 208
 209 // Called from the main cast thread.
 210 void VideoReceiver::GetEncodedVideoFrame(
 211     const VideoFrameEncodedCallback& callback) {
 212   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 213   scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
 214       new transport::EncodedVideoFrame());
 215   uint32 rtp_timestamp = 0;
 216   bool next_frame = false;
 217
 218   if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
 219                                      &next_frame)) {
 220     // We have no video frames. Wait for new packet(s).
 221     queued_encoded_callbacks_.push_back(callback);
 222     return;
 223   }
 224
 225   if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) {
 226     // Logging already done.
 227     queued_encoded_callbacks_.push_back(callback);
 228     return;
 229   }
 230
 231   base::TimeTicks render_time;
 232   if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
 233                             &render_time)) {
 234     cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 235         base::Bind(callback, base::Passed(&encoded_frame), render_time));
 236   } else {
 237     // We have a video frame; however we are missing packets and we have time
 238     // to wait for new packet(s).
 239     queued_encoded_callbacks_.push_back(callback);
 240   }
 241 }
 242
 243 // Should we pull the encoded video frame from the framer? decided by if this is
 244 // the next frame or we are running out of time and have to pull the following
 245 // frame.
 246 // If the frame is too old to be rendered we set the don't show flag in the
 247 // video bitstream where possible.
 248 bool VideoReceiver::PullEncodedVideoFrame(uint32 rtp_timestamp,
 249     bool next_frame, scoped_ptr<transport::EncodedVideoFrame>* encoded_frame,
 250     base::TimeTicks* render_time) {
 251   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 252   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 253   *render_time = GetRenderTime(now, rtp_timestamp);
 254
 255   // TODO(mikhal): Store actual render time and not diff.
 256   cast_environment_->Logging()->InsertFrameEventWithDelay(now,
 257       kVideoRenderDelay, rtp_timestamp, (*encoded_frame)->frame_id,
 258       now - *render_time);
 259
 260   // Minimum time before a frame is due to be rendered before we pull it for
 261   // decode.
 262   base::TimeDelta min_wait_delta = frame_delay_;
 263   base::TimeDelta time_until_render = *render_time - now;
 264   if (!next_frame && (time_until_render > min_wait_delta)) {
 265     // Example:
 266     // We have decoded frame 1 and we have received the complete frame 3, but
 267     // not frame 2. If we still have time before frame 3 should be rendered we
 268     // will wait for 2 to arrive, however if 2 never show up this timer will hit
 269     // and we will pull out frame 3 for decoding and rendering.
 270     base::TimeDelta time_until_release = time_until_render - min_wait_delta;
 271     cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 272         base::Bind(&VideoReceiver::PlayoutTimeout, weak_factory_.GetWeakPtr()),
 273         time_until_release);
 274     VLOG(1) << "Wait before releasing frame "
 275             << static_cast<int>((*encoded_frame)->frame_id)
 276             << " time " << time_until_release.InMilliseconds();
 277     return false;
 278   }
 279
 280   base::TimeDelta dont_show_timeout_delta =
 281       base::TimeDelta::FromMilliseconds(-kDontShowTimeoutMs);
 282   if (codec_ == transport::kVp8 &&
 283       time_until_render < dont_show_timeout_delta) {
 284     (*encoded_frame)->data[0] &= 0xef;
 285     VLOG(1) << "Don't show frame "
 286             << static_cast<int>((*encoded_frame)->frame_id)
 287             << " time_until_render:" << time_until_render.InMilliseconds();
 288   } else {
 289     VLOG(1) << "Show frame "
 290             << static_cast<int>((*encoded_frame)->frame_id)
 291             << " time_until_render:" << time_until_render.InMilliseconds();
 292   }
 293   // We have a copy of the frame, release this one.
 294   framer_->ReleaseFrame((*encoded_frame)->frame_id);
 295   (*encoded_frame)->codec = codec_;
 296   return true;
 297 }
 298
 299 void VideoReceiver::PlayoutTimeout() {
 300   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 301   if (queued_encoded_callbacks_.empty()) return;
 302
 303   uint32 rtp_timestamp = 0;
 304   bool next_frame = false;
 305   scoped_ptr<transport::EncodedVideoFrame> encoded_frame(
 306       new transport::EncodedVideoFrame());
 307
 308   if (!framer_->GetEncodedVideoFrame(encoded_frame.get(), &rtp_timestamp,
 309                                      &next_frame)) {
 310     // We have no video frames. Wait for new packet(s).
 311     // Since the application can post multiple VideoFrameEncodedCallback and
 312     // we only check the next frame to play out we might have multiple timeout
 313     // events firing after each other; however this should be a rare event.
 314     VLOG(1) << "Failed to retrieved a complete frame at this point in time";
 315     return;
 316   }
 317   VLOG(1) << "PlayoutTimeout retrieved frame "
 318           << static_cast<int>(encoded_frame->frame_id);
 319
 320   if (decryptor_ && !DecryptVideoFrame(&encoded_frame)) {
 321     // Logging already done.
 322     return;
 323   }
 324
 325   base::TimeTicks render_time;
 326   if (PullEncodedVideoFrame(rtp_timestamp, next_frame, &encoded_frame,
 327                             &render_time)) {
 328     if (!queued_encoded_callbacks_.empty()) {
 329       VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 330       queued_encoded_callbacks_.pop_front();
 331       cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 332           base::Bind(callback, base::Passed(&encoded_frame), render_time));
 333     }
 334   } else {
 335     // We have a video frame; however we are missing packets and we have time
 336     // to wait for new packet(s).
 337   }
 338 }
 339
 340 base::TimeTicks VideoReceiver::GetRenderTime(base::TimeTicks now,
 341                                              uint32 rtp_timestamp) {
 342   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 343   // Senders time in ms when this frame was captured.
 344   // Note: the senders clock and our local clock might not be synced.
 345   base::TimeTicks rtp_timestamp_in_ticks;
 346
 347   if (time_offset_.InMilliseconds() == 0) {
 348     if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 349                                          incoming_rtp_timestamp_,
 350                                          &rtp_timestamp_in_ticks)) {
 351       // We have not received any RTCP to sync the stream play it out as soon as
 352       // possible.
 353       return now;
 354     }
 355     time_offset_ = time_incoming_packet_ - rtp_timestamp_in_ticks;
 356   } else if (time_incoming_packet_updated_) {
 357     if (rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 358                                         incoming_rtp_timestamp_,
 359                                         &rtp_timestamp_in_ticks)) {
 360       // Time to update the time_offset.
 361       base::TimeDelta time_offset =
 362           time_incoming_packet_ - rtp_timestamp_in_ticks;
 363       time_offset_ = ((kTimeOffsetFilter - 1) * time_offset_ + time_offset)
 364           / kTimeOffsetFilter;
 365     }
 366   }
 367   // Reset |time_incoming_packet_updated_| to enable a future measurement.
 368   time_incoming_packet_updated_ = false;
 369   if (!rtcp_->RtpTimestampInSenderTime(kVideoFrequency,
 370                                        rtp_timestamp,
 371                                        &rtp_timestamp_in_ticks)) {
 372     // This can fail if we have not received any RTCP packets in a long time.
 373     return now;
 374   }
 375   base::TimeTicks render_time =
 376       rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_;
 377   if (last_render_time_ > render_time)
 378     render_time = last_render_time_;
 379   last_render_time_ = render_time;
 380   return render_time;
 381 }
 382
 383 void VideoReceiver::IncomingPacket(const uint8* packet, size_t length,
 384                                    const base::Closure callback) {
 385   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 386   if (Rtcp::IsRtcpPacket(packet, length)) {
 387     rtcp_->IncomingRtcpPacket(packet, length);
 388   } else {
 389     rtp_receiver_.ReceivedPacket(packet, length);
 390   }
 391   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE, callback);
 392 }
 393
 394 void VideoReceiver::IncomingParsedRtpPacket(const uint8* payload_data,
 395                                             size_t payload_size,
 396                                             const RtpCastHeader& rtp_header) {
 397   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 398
 399   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 400   if (time_incoming_packet_.is_null() || now - time_incoming_packet_ >
 401       base::TimeDelta::FromMilliseconds(kMinTimeBetweenOffsetUpdatesMs)) {
 402     if (time_incoming_packet_.is_null()) InitializeTimers();
 403     incoming_rtp_timestamp_ = rtp_header.webrtc.header.timestamp;
 404     time_incoming_packet_ = now;
 405     time_incoming_packet_updated_ = true;
 406   }
 407
 408   cast_environment_->Logging()->InsertPacketEvent(now, kPacketReceived,
 409       rtp_header.webrtc.header.timestamp, rtp_header.frame_id,
 410       rtp_header.packet_id, rtp_header.max_packet_id, payload_size);
 411
 412   bool duplicate = false;
 413   bool complete = framer_->InsertPacket(payload_data, payload_size, rtp_header,
 414                                         &duplicate);
 415
 416   if (duplicate) {
 417     cast_environment_->Logging()->InsertPacketEvent(now,
 418         kDuplicatePacketReceived,
 419         rtp_header.webrtc.header.timestamp, rtp_header.frame_id,
 420         rtp_header.packet_id, rtp_header.max_packet_id, payload_size);
 421     // Duplicate packets are ignored.
 422     return;
 423   }
 424   if (!complete) return;  // Video frame not complete; wait for more packets.
 425   if (queued_encoded_callbacks_.empty()) return;  // No pending callback.
 426
 427   VideoFrameEncodedCallback callback = queued_encoded_callbacks_.front();
 428   queued_encoded_callbacks_.pop_front();
 429   cast_environment_->PostTask(CastEnvironment::MAIN, FROM_HERE,
 430       base::Bind(&VideoReceiver::GetEncodedVideoFrame,
 431           weak_factory_.GetWeakPtr(), callback));
 432 }
 433
 434 // Send a cast feedback message. Actual message created in the framer (cast
 435 // message builder).
 436 void VideoReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
 437   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 438   // TODO(pwestin): wire up log messages.
 439   rtcp_->SendRtcpFromRtpReceiver(&cast_message, NULL);
 440   time_last_sent_cast_message_= cast_environment_->Clock()->NowTicks();
 441 }
 442
 443 // Cast messages should be sent within a maximum interval. Schedule a call
 444 // if not triggered elsewhere, e.g. by the cast message_builder.
 445 void VideoReceiver::ScheduleNextCastMessage() {
 446   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 447   base::TimeTicks send_time;
 448   framer_->TimeToSendNextCastMessage(&send_time);
 449
 450   base::TimeDelta time_to_send = send_time -
 451       cast_environment_->Clock()->NowTicks();
 452   time_to_send = std::max(time_to_send,
 453       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 454   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 455       base::Bind(&VideoReceiver::SendNextCastMessage,
 456                  weak_factory_.GetWeakPtr()), time_to_send);
 457 }
 458
 459 void VideoReceiver::SendNextCastMessage() {
 460   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 461   framer_->SendCastMessage();  // Will only send a message if it is time.
 462   ScheduleNextCastMessage();
 463 }
 464
 465 // Schedule the next RTCP report to be sent back to the sender.
 466 void VideoReceiver::ScheduleNextRtcpReport() {
 467   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 468   base::TimeDelta time_to_next = rtcp_->TimeToSendNextRtcpReport() -
 469       cast_environment_->Clock()->NowTicks();
 470
 471   time_to_next = std::max(time_to_next,
 472       base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 473
 474   cast_environment_->PostDelayedTask(CastEnvironment::MAIN, FROM_HERE,
 475       base::Bind(&VideoReceiver::SendNextRtcpReport,
 476                 weak_factory_.GetWeakPtr()), time_to_next);
 477 }
 478
 479 void VideoReceiver::SendNextRtcpReport() {
 480   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 481   rtcp_->SendRtcpFromRtpReceiver(NULL, NULL);
 482   ScheduleNextRtcpReport();
 483 }
 484
 485 }  // namespace cast
 486 }  // namespace media