media/formats/mp2t/es_adapter_video.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/formats/mp2t/es_adapter_video.h"
   6
   7 #include "media/base/buffers.h"
   8 #include "media/base/stream_parser_buffer.h"
   9 #include "media/base/video_decoder_config.h"
  10 #include "media/formats/mp2t/mp2t_common.h"
  11
  12 namespace media {
  13 namespace mp2t {
  14
  15 // Arbitrary decision about the frame duration when there is no previous
  16 // hint about what could be the frame duration.
  17 static const int kDefaultFrameDurationMs = 40;
  18
  19 // To calculate the frame duration, we make an assumption
  20 // that the timestamp of the next frame in presentation order
  21 // is no further than 5 frames away in decode order.
  22 // TODO(damienv): the previous assumption should cover most of the practical
  23 // cases. However, the right way to calculate the frame duration would be
  24 // to emulate the H264 dpb bumping process.
  25 static const size_t kHistorySize = 5;
  26
  27 EsAdapterVideo::EsAdapterVideo(
  28     const NewVideoConfigCB& new_video_config_cb,
  29     const EmitBufferCB& emit_buffer_cb)
  30     : new_video_config_cb_(new_video_config_cb),
  31       emit_buffer_cb_(emit_buffer_cb),
  32       has_valid_config_(false),
  33       has_valid_frame_(false),
  34       last_frame_duration_(
  35           base::TimeDelta::FromMilliseconds(kDefaultFrameDurationMs)),
  36       buffer_index_(0),
  37       has_valid_initial_timestamp_(false),
  38       discarded_frame_count_(0) {
  39 }
  40
  41 EsAdapterVideo::~EsAdapterVideo() {
  42 }
  43
  44 void EsAdapterVideo::Flush() {
  45   ProcessPendingBuffers(true);
  46 }
  47
  48 void EsAdapterVideo::Reset() {
  49   has_valid_config_ = false;
  50   has_valid_frame_ = false;
  51
  52   last_frame_duration_ =
  53       base::TimeDelta::FromMilliseconds(kDefaultFrameDurationMs);
  54
  55   config_list_.clear();
  56   buffer_index_ = 0;
  57   buffer_list_.clear();
  58   emitted_pts_.clear();
  59
  60   has_valid_initial_timestamp_ = false;
  61   min_pts_ = base::TimeDelta();
  62   min_dts_ = DecodeTimestamp();
  63
  64   discarded_frame_count_ = 0;
  65 }
  66
  67 void EsAdapterVideo::OnConfigChanged(
  68     const VideoDecoderConfig& video_decoder_config) {
  69   config_list_.push_back(
  70       ConfigEntry(buffer_index_ + buffer_list_.size(), video_decoder_config));
  71   has_valid_config_ = true;
  72   ProcessPendingBuffers(false);
  73 }
  74
  75 bool EsAdapterVideo::OnNewBuffer(
  76     const scoped_refptr<StreamParserBuffer>& stream_parser_buffer) {
  77   if (stream_parser_buffer->timestamp() == kNoTimestamp()) {
  78     if (has_valid_frame_) {
  79       // There is currently no error concealment for a missing timestamp
  80       // in the middle of the stream.
  81       DVLOG(1) << "Missing timestamp in the middle of the stream";
  82       return false;
  83     }
  84
  85     if (!has_valid_initial_timestamp_) {
  86       // MPEG-2 TS requires the first access unit to be given a timestamp.
  87       // However, some streams do not comply with this requirement.
  88       // So simply drop the frame if it is a leading frame with no timestamp.
  89       DVLOG(1)
  90           << "Stream not compliant: ignoring leading frame with no timestamp";
  91       return true;
  92     }
  93
  94     // In all the other cases, this frame will be replaced by the following
  95     // valid key frame, using timestamp interpolation.
  96     DCHECK(has_valid_initial_timestamp_);
  97     DCHECK_GE(discarded_frame_count_, 1);
  98     discarded_frame_count_++;
  99     return true;
 100   }
 101
 102   // At this point, timestamps of the incoming frame are valid.
 103   if (!has_valid_initial_timestamp_) {
 104     min_pts_ = stream_parser_buffer->timestamp();
 105     min_dts_ = stream_parser_buffer->GetDecodeTimestamp();
 106     has_valid_initial_timestamp_ = true;
 107   }
 108   if (stream_parser_buffer->timestamp() < min_pts_)
 109     min_pts_ = stream_parser_buffer->timestamp();
 110
 111   // Discard the incoming frame:
 112   // - if it is not associated with any config,
 113   // - or if no valid key frame has been found so far.
 114   if (!has_valid_config_ ||
 115       (!has_valid_frame_ && !stream_parser_buffer->is_key_frame())) {
 116     discarded_frame_count_++;
 117     return true;
 118   }
 119
 120   has_valid_frame_ = true;
 121
 122   if (discarded_frame_count_ > 0)
 123     ReplaceDiscardedFrames(stream_parser_buffer);
 124
 125   buffer_list_.push_back(stream_parser_buffer);
 126   ProcessPendingBuffers(false);
 127   return true;
 128 }
 129
 130 void EsAdapterVideo::ProcessPendingBuffers(bool flush) {
 131   DCHECK(has_valid_config_);
 132
 133   while (!buffer_list_.empty() &&
 134          (flush || buffer_list_.size() > kHistorySize)) {
 135     // Signal a config change, just before emitting the corresponding frame.
 136     if (!config_list_.empty() && config_list_.front().first == buffer_index_) {
 137       new_video_config_cb_.Run(config_list_.front().second);
 138       config_list_.pop_front();
 139     }
 140
 141     scoped_refptr<StreamParserBuffer> buffer = buffer_list_.front();
 142     buffer_list_.pop_front();
 143     buffer_index_++;
 144
 145     if (buffer->duration() == kNoTimestamp()) {
 146       base::TimeDelta next_frame_pts = GetNextFramePts(buffer->timestamp());
 147       if (next_frame_pts == kNoTimestamp()) {
 148         // This can happen when emitting the very last buffer
 149         // or if the stream do not meet the assumption behind |kHistorySize|.
 150         DVLOG(LOG_LEVEL_ES) << "Using last frame duration: "
 151                             << last_frame_duration_.InMilliseconds();
 152         buffer->set_duration(last_frame_duration_);
 153       } else {
 154         base::TimeDelta duration = next_frame_pts - buffer->timestamp();
 155         DVLOG(LOG_LEVEL_ES) << "Frame duration: " << duration.InMilliseconds();
 156         buffer->set_duration(duration);
 157       }
 158     }
 159
 160     emitted_pts_.push_back(buffer->timestamp());
 161     if (emitted_pts_.size() > kHistorySize)
 162       emitted_pts_.pop_front();
 163
 164     last_frame_duration_ = buffer->duration();
 165     emit_buffer_cb_.Run(buffer);
 166   }
 167 }
 168
 169 base::TimeDelta EsAdapterVideo::GetNextFramePts(base::TimeDelta current_pts) {
 170   base::TimeDelta next_pts = kNoTimestamp();
 171
 172   // Consider the timestamps of future frames (in decode order).
 173   // Note: the next frame is not enough when the GOP includes some B frames.
 174   for (BufferQueue::const_iterator it = buffer_list_.begin();
 175        it != buffer_list_.end(); ++it) {
 176     if ((*it)->timestamp() < current_pts)
 177       continue;
 178     if (next_pts == kNoTimestamp() || next_pts > (*it)->timestamp())
 179       next_pts = (*it)->timestamp();
 180   }
 181
 182   // Consider the timestamps of previous frames (in decode order).
 183   // In a simple GOP structure with B frames, the frame next to the last B
 184   // frame (in presentation order) is located before in decode order.
 185   for (std::list<base::TimeDelta>::const_iterator it = emitted_pts_.begin();
 186        it != emitted_pts_.end(); ++it) {
 187     if (*it < current_pts)
 188       continue;
 189     if (next_pts == kNoTimestamp() || next_pts > *it)
 190       next_pts = *it;
 191   }
 192
 193   return next_pts;
 194 }
 195
 196 void EsAdapterVideo::ReplaceDiscardedFrames(
 197     const scoped_refptr<StreamParserBuffer>& stream_parser_buffer) {
 198   DCHECK_GT(discarded_frame_count_, 0);
 199   DCHECK(stream_parser_buffer->is_key_frame());
 200
 201   // PTS/DTS are interpolated between the min PTS/DTS of discarded frames
 202   // and the PTS/DTS of the first valid buffer.
 203   // Note: |pts_delta| and |dts_delta| are calculated using integer division.
 204   // Interpolation thus accumulutes small errors. However, since timestamps
 205   // are given in microseconds, only a high number of discarded frames
 206   // (in the order of 10000s) could have an impact and create a gap (from MSE
 207   // point of view) between the last interpolated frame and
 208   // |stream_parser_buffer|.
 209   base::TimeDelta pts = min_pts_;
 210   base::TimeDelta pts_delta =
 211       (stream_parser_buffer->timestamp() - pts) / discarded_frame_count_;
 212   DecodeTimestamp dts = min_dts_;
 213   base::TimeDelta dts_delta =
 214       (stream_parser_buffer->GetDecodeTimestamp() - dts) /
 215       discarded_frame_count_;
 216
 217   for (int i = 0; i < discarded_frame_count_; i++) {
 218     scoped_refptr<StreamParserBuffer> frame =
 219         StreamParserBuffer::CopyFrom(
 220             stream_parser_buffer->data(),
 221             stream_parser_buffer->data_size(),
 222             stream_parser_buffer->is_key_frame(),
 223             stream_parser_buffer->type(),
 224             stream_parser_buffer->track_id());
 225     frame->SetDecodeTimestamp(dts);
 226     frame->set_timestamp(pts);
 227     frame->set_duration(pts_delta);
 228     buffer_list_.push_back(frame);
 229     pts += pts_delta;
 230     dts += dts_delta;
 231   }
 232   discarded_frame_count_ = 0;
 233 }
 234
 235 }  // namespace mp2t
 236 }  // namespace media