media/cast/audio_receiver/audio_receiver.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/cast/audio_receiver/audio_receiver.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/logging.h"
  11 #include "base/message_loop/message_loop.h"
  12 #include "media/cast/audio_receiver/audio_decoder.h"
  13 #include "media/cast/transport/cast_transport_defines.h"
  14
  15 namespace {
  16 const int kMinSchedulingDelayMs = 1;
  17 // TODO(miu): This should go in AudioReceiverConfig.
  18 const int kTypicalAudioFrameDurationMs = 10;
  19 }  // namespace
  20
  21 namespace media {
  22 namespace cast {
  23
  24 AudioReceiver::AudioReceiver(scoped_refptr<CastEnvironment> cast_environment,
  25                              const AudioReceiverConfig& audio_config,
  26                              transport::PacedPacketSender* const packet_sender)
  27     : RtpReceiver(cast_environment->Clock(), &audio_config, NULL),
  28       cast_environment_(cast_environment),
  29       event_subscriber_(kReceiverRtcpEventHistorySize, AUDIO_EVENT),
  30       codec_(audio_config.codec),
  31       frequency_(audio_config.frequency),
  32       target_delay_delta_(
  33           base::TimeDelta::FromMilliseconds(audio_config.rtp_max_delay_ms)),
  34       framer_(cast_environment->Clock(),
  35               this,
  36               audio_config.incoming_ssrc,
  37               true,
  38               audio_config.rtp_max_delay_ms / kTypicalAudioFrameDurationMs),
  39       rtcp_(cast_environment,
  40             NULL,
  41             NULL,
  42             packet_sender,
  43             GetStatistics(),
  44             audio_config.rtcp_mode,
  45             base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval),
  46             audio_config.feedback_ssrc,
  47             audio_config.incoming_ssrc,
  48             audio_config.rtcp_c_name,
  49             true),
  50       is_waiting_for_consecutive_frame_(false),
  51       weak_factory_(this) {
  52   if (!audio_config.use_external_decoder)
  53     audio_decoder_.reset(new AudioDecoder(cast_environment, audio_config));
  54   decryptor_.Initialize(audio_config.aes_key, audio_config.aes_iv_mask);
  55   rtcp_.SetTargetDelay(target_delay_delta_);
  56   cast_environment_->Logging()->AddRawEventSubscriber(&event_subscriber_);
  57   memset(frame_id_to_rtp_timestamp_, 0, sizeof(frame_id_to_rtp_timestamp_));
  58 }
  59
  60 AudioReceiver::~AudioReceiver() {
  61   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  62   cast_environment_->Logging()->RemoveRawEventSubscriber(&event_subscriber_);
  63 }
  64
  65 void AudioReceiver::InitializeTimers() {
  66   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  67   ScheduleNextRtcpReport();
  68   ScheduleNextCastMessage();
  69 }
  70
  71 void AudioReceiver::OnReceivedPayloadData(const uint8* payload_data,
  72                                           size_t payload_size,
  73                                           const RtpCastHeader& rtp_header) {
  74   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
  75   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
  76
  77   // TODO(pwestin): update this as video to refresh over time.
  78   if (time_first_incoming_packet_.is_null()) {
  79     InitializeTimers();
  80     first_incoming_rtp_timestamp_ = rtp_header.rtp_timestamp;
  81     time_first_incoming_packet_ = now;
  82   }
  83
  84   frame_id_to_rtp_timestamp_[rtp_header.frame_id & 0xff] =
  85       rtp_header.rtp_timestamp;
  86   cast_environment_->Logging()->InsertPacketEvent(
  87       now, PACKET_RECEIVED, AUDIO_EVENT, rtp_header.rtp_timestamp,
  88       rtp_header.frame_id, rtp_header.packet_id, rtp_header.max_packet_id,
  89       payload_size);
  90
  91   bool duplicate = false;
  92   const bool complete =
  93       framer_.InsertPacket(payload_data, payload_size, rtp_header, &duplicate);
  94
  95   // Duplicate packets are ignored.
  96   if (duplicate || !complete)
  97     return;
  98
  99   EmitAvailableEncodedFrames();
 100 }
 101
 102 void AudioReceiver::GetRawAudioFrame(
 103     const AudioFrameDecodedCallback& callback) {
 104   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 105   DCHECK(!callback.is_null());
 106   DCHECK(audio_decoder_.get());
 107   GetEncodedAudioFrame(base::Bind(
 108       &AudioReceiver::DecodeEncodedAudioFrame,
 109       // Note: Use of Unretained is safe since this Closure is guaranteed to be
 110       // invoked before destruction of |this|.
 111       base::Unretained(this),
 112       callback));
 113 }
 114
 115 void AudioReceiver::DecodeEncodedAudioFrame(
 116     const AudioFrameDecodedCallback& callback,
 117     scoped_ptr<transport::EncodedFrame> encoded_frame) {
 118   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 119   if (!encoded_frame) {
 120     callback.Run(make_scoped_ptr<AudioBus>(NULL), base::TimeTicks(), false);
 121     return;
 122   }
 123   const uint32 frame_id = encoded_frame->frame_id;
 124   const uint32 rtp_timestamp = encoded_frame->rtp_timestamp;
 125   const base::TimeTicks playout_time = encoded_frame->reference_time;
 126   audio_decoder_->DecodeFrame(encoded_frame.Pass(),
 127                               base::Bind(&AudioReceiver::EmitRawAudioFrame,
 128                                          cast_environment_,
 129                                          callback,
 130                                          frame_id,
 131                                          rtp_timestamp,
 132                                          playout_time));
 133 }
 134
 135 // static
 136 void AudioReceiver::EmitRawAudioFrame(
 137     const scoped_refptr<CastEnvironment>& cast_environment,
 138     const AudioFrameDecodedCallback& callback,
 139     uint32 frame_id,
 140     uint32 rtp_timestamp,
 141     const base::TimeTicks& playout_time,
 142     scoped_ptr<AudioBus> audio_bus,
 143     bool is_continuous) {
 144   DCHECK(cast_environment->CurrentlyOn(CastEnvironment::MAIN));
 145   if (audio_bus.get()) {
 146     const base::TimeTicks now = cast_environment->Clock()->NowTicks();
 147     cast_environment->Logging()->InsertFrameEvent(
 148         now, FRAME_DECODED, AUDIO_EVENT, rtp_timestamp, frame_id);
 149     cast_environment->Logging()->InsertFrameEventWithDelay(
 150         now, FRAME_PLAYOUT, AUDIO_EVENT, rtp_timestamp, frame_id,
 151         playout_time - now);
 152   }
 153   callback.Run(audio_bus.Pass(), playout_time, is_continuous);
 154 }
 155
 156 void AudioReceiver::GetEncodedAudioFrame(const FrameEncodedCallback& callback) {
 157   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 158   frame_request_queue_.push_back(callback);
 159   EmitAvailableEncodedFrames();
 160 }
 161
 162 void AudioReceiver::EmitAvailableEncodedFrames() {
 163   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 164
 165   while (!frame_request_queue_.empty()) {
 166     // Attempt to peek at the next completed frame from the |framer_|.
 167     // TODO(miu): We should only be peeking at the metadata, and not copying the
 168     // payload yet!  Or, at least, peek using a StringPiece instead of a copy.
 169     scoped_ptr<transport::EncodedFrame> encoded_frame(
 170         new transport::EncodedFrame());
 171     bool is_consecutively_next_frame = false;
 172     if (!framer_.GetEncodedAudioFrame(encoded_frame.get(),
 173                                       &is_consecutively_next_frame)) {
 174       VLOG(1) << "Wait for more audio packets to produce a completed frame.";
 175       return;  // OnReceivedPayloadData() will invoke this method in the future.
 176     }
 177
 178     // If |framer_| has a frame ready that is out of sequence, examine the
 179     // playout time to determine whether it's acceptable to continue, thereby
 180     // skipping one or more frames.  Skip if the missing frame wouldn't complete
 181     // playing before the start of playback of the available frame.
 182     const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 183     const base::TimeTicks playout_time =
 184         GetPlayoutTime(now, encoded_frame->rtp_timestamp);
 185     if (!is_consecutively_next_frame) {
 186       // TODO(miu): Also account for expected decode time here?
 187       const base::TimeTicks earliest_possible_end_time_of_missing_frame =
 188           now + base::TimeDelta::FromMilliseconds(kTypicalAudioFrameDurationMs);
 189       if (earliest_possible_end_time_of_missing_frame < playout_time) {
 190         VLOG(1) << "Wait for next consecutive frame instead of skipping.";
 191         if (!is_waiting_for_consecutive_frame_) {
 192           is_waiting_for_consecutive_frame_ = true;
 193           cast_environment_->PostDelayedTask(
 194               CastEnvironment::MAIN,
 195               FROM_HERE,
 196               base::Bind(&AudioReceiver::EmitAvailableEncodedFramesAfterWaiting,
 197                          weak_factory_.GetWeakPtr()),
 198               playout_time - now);
 199         }
 200         return;
 201       }
 202     }
 203
 204     // Decrypt the payload data in the frame, if crypto is being used.
 205     if (decryptor_.initialized()) {
 206       std::string decrypted_audio_data;
 207       if (!decryptor_.Decrypt(encoded_frame->frame_id,
 208                               encoded_frame->data,
 209                               &decrypted_audio_data)) {
 210         // Decryption failed.  Give up on this frame, releasing it from the
 211         // jitter buffer.
 212         framer_.ReleaseFrame(encoded_frame->frame_id);
 213         continue;
 214       }
 215       encoded_frame->data.swap(decrypted_audio_data);
 216     }
 217
 218     // At this point, we have a decrypted EncodedFrame ready to be emitted.
 219     encoded_frame->reference_time = playout_time;
 220     framer_.ReleaseFrame(encoded_frame->frame_id);
 221     cast_environment_->PostTask(CastEnvironment::MAIN,
 222                                 FROM_HERE,
 223                                 base::Bind(frame_request_queue_.front(),
 224                                            base::Passed(&encoded_frame)));
 225     frame_request_queue_.pop_front();
 226   }
 227 }
 228
 229 void AudioReceiver::EmitAvailableEncodedFramesAfterWaiting() {
 230   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 231   DCHECK(is_waiting_for_consecutive_frame_);
 232   is_waiting_for_consecutive_frame_ = false;
 233   EmitAvailableEncodedFrames();
 234 }
 235
 236 void AudioReceiver::IncomingPacket(scoped_ptr<Packet> packet) {
 237   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 238   if (Rtcp::IsRtcpPacket(&packet->front(), packet->size())) {
 239     rtcp_.IncomingRtcpPacket(&packet->front(), packet->size());
 240   } else {
 241     ReceivedPacket(&packet->front(), packet->size());
 242   }
 243 }
 244
 245 void AudioReceiver::SetTargetDelay(base::TimeDelta target_delay) {
 246   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 247   target_delay_delta_ = target_delay;
 248   rtcp_.SetTargetDelay(target_delay_delta_);
 249 }
 250
 251 void AudioReceiver::CastFeedback(const RtcpCastMessage& cast_message) {
 252   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 253   base::TimeTicks now = cast_environment_->Clock()->NowTicks();
 254   RtpTimestamp rtp_timestamp =
 255       frame_id_to_rtp_timestamp_[cast_message.ack_frame_id_ & 0xff];
 256   cast_environment_->Logging()->InsertFrameEvent(
 257       now, FRAME_ACK_SENT, AUDIO_EVENT, rtp_timestamp,
 258       cast_message.ack_frame_id_);
 259
 260   ReceiverRtcpEventSubscriber::RtcpEventMultiMap rtcp_events;
 261   event_subscriber_.GetRtcpEventsAndReset(&rtcp_events);
 262   rtcp_.SendRtcpFromRtpReceiver(&cast_message, &rtcp_events);
 263 }
 264
 265 base::TimeTicks AudioReceiver::GetPlayoutTime(base::TimeTicks now,
 266                                               uint32 rtp_timestamp) {
 267   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 268   // Senders time in ms when this frame was recorded.
 269   // Note: the senders clock and our local clock might not be synced.
 270   base::TimeTicks rtp_timestamp_in_ticks;
 271   base::TimeTicks playout_time;
 272   if (time_offset_ == base::TimeDelta()) {
 273     if (rtcp_.RtpTimestampInSenderTime(frequency_,
 274                                        first_incoming_rtp_timestamp_,
 275                                        &rtp_timestamp_in_ticks)) {
 276       time_offset_ = time_first_incoming_packet_ - rtp_timestamp_in_ticks;
 277       // TODO(miu): As clocks drift w.r.t. each other, and other factors take
 278       // effect, |time_offset_| should be updated.  Otherwise, we might as well
 279       // always compute the time offsets agnostic of RTCP's time data.
 280     } else {
 281       // We have not received any RTCP to sync the stream play it out as soon as
 282       // possible.
 283
 284       // BUG: This means we're literally switching to a different timeline a
 285       // short time after a cast receiver has been running.  Re-enable
 286       // End2EndTest.StartSenderBeforeReceiver once this is fixed.
 287       // http://crbug.com/356942
 288       uint32 rtp_timestamp_diff = rtp_timestamp - first_incoming_rtp_timestamp_;
 289
 290       int frequency_khz = frequency_ / 1000;
 291       base::TimeDelta rtp_time_diff_delta =
 292           base::TimeDelta::FromMilliseconds(rtp_timestamp_diff / frequency_khz);
 293       base::TimeDelta time_diff_delta = now - time_first_incoming_packet_;
 294
 295       playout_time = now + std::max(rtp_time_diff_delta - time_diff_delta,
 296                                     base::TimeDelta());
 297     }
 298   }
 299   if (playout_time.is_null()) {
 300     // This can fail if we have not received any RTCP packets in a long time.
 301     if (rtcp_.RtpTimestampInSenderTime(frequency_, rtp_timestamp,
 302                                        &rtp_timestamp_in_ticks)) {
 303       playout_time =
 304           rtp_timestamp_in_ticks + time_offset_ + target_delay_delta_;
 305     } else {
 306       playout_time = now;
 307     }
 308   }
 309
 310   // TODO(miu): This is broken since we literally switch timelines once |rtcp_|
 311   // can provide us the |time_offset_|.  Furthermore, this "getter" method may
 312   // be called on frames received out-of-order, which means the playout times
 313   // for earlier frames will be computed incorrectly.
 314 #if 0
 315   // Don't allow the playout time to go backwards.
 316   if (last_playout_time_ > playout_time) playout_time = last_playout_time_;
 317   last_playout_time_ = playout_time;
 318 #endif
 319
 320   return playout_time;
 321 }
 322
 323 void AudioReceiver::ScheduleNextRtcpReport() {
 324   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 325   base::TimeDelta time_to_send = rtcp_.TimeToSendNextRtcpReport() -
 326                                  cast_environment_->Clock()->NowTicks();
 327
 328   time_to_send = std::max(
 329       time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 330
 331   cast_environment_->PostDelayedTask(
 332       CastEnvironment::MAIN,
 333       FROM_HERE,
 334       base::Bind(&AudioReceiver::SendNextRtcpReport,
 335                  weak_factory_.GetWeakPtr()),
 336       time_to_send);
 337 }
 338
 339 void AudioReceiver::SendNextRtcpReport() {
 340   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 341   // TODO(pwestin): add logging.
 342   rtcp_.SendRtcpFromRtpReceiver(NULL, NULL);
 343   ScheduleNextRtcpReport();
 344 }
 345
 346 // Cast messages should be sent within a maximum interval. Schedule a call
 347 // if not triggered elsewhere, e.g. by the cast message_builder.
 348 void AudioReceiver::ScheduleNextCastMessage() {
 349   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 350   base::TimeTicks send_time;
 351   framer_.TimeToSendNextCastMessage(&send_time);
 352   base::TimeDelta time_to_send =
 353       send_time - cast_environment_->Clock()->NowTicks();
 354   time_to_send = std::max(
 355       time_to_send, base::TimeDelta::FromMilliseconds(kMinSchedulingDelayMs));
 356   cast_environment_->PostDelayedTask(
 357       CastEnvironment::MAIN,
 358       FROM_HERE,
 359       base::Bind(&AudioReceiver::SendNextCastMessage,
 360                  weak_factory_.GetWeakPtr()),
 361       time_to_send);
 362 }
 363
 364 void AudioReceiver::SendNextCastMessage() {
 365   DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
 366   framer_.SendCastMessage();  // Will only send a message if it is time.
 367   ScheduleNextCastMessage();
 368 }
 369
 370 }  // namespace cast
 371 }  // namespace media