content/renderer/media/video_track_recorder.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_recorder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/sys_info.h"
  10 #include "base/threading/thread.h"
  11 #include "base/time/time.h"
  12 #include "base/trace_event/trace_event.h"
  13 #include "content/child/child_process.h"
  14 #include "media/base/bind_to_current_loop.h"
  15 #include "media/base/video_frame.h"
  16
  17 extern "C" {
  18 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
  19 // backwards compatibility for legacy applications using the library.
  20 #define VPX_CODEC_DISABLE_COMPAT 1
  21 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  22 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
  23 }
  24
  25 using media::VideoFrame;
  26 using media::VideoFrameMetadata;
  27
  28 namespace content {
  29
  30 namespace {
  31 const vpx_codec_flags_t kNoFlags = 0;
  32
  33 void OnFrameEncodeCompleted(
  34     const content::VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
  35     const scoped_refptr<VideoFrame>& frame,
  36     scoped_ptr<std::string> data,
  37     base::TimeTicks capture_timestamp,
  38     bool keyframe) {
  39   DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "
  40            << capture_timestamp << " ms - " << data->length() << "B ";
  41   on_encoded_video_cb.Run(frame, base::StringPiece(*data), capture_timestamp,
  42                           keyframe);
  43 }
  44
  45 }  // anonymous namespace
  46
  47 // Inner class encapsulating all libvpx interactions and the encoding+delivery
  48 // of received frames. This class is:
  49 // - created and destroyed on its parent's thread (usually the main render
  50 // thread),
  51 // - receives VideoFrames and Run()s the callbacks on another thread (supposedly
  52 // the render IO thread), which is cached on first frame arrival,
  53 // - uses an internal |encoding_thread_| for libvpx interactions, notably for
  54 // encoding (which might take some time).
  55 // Only VP8 is supported for the time being.
  56 class VideoTrackRecorder::VpxEncoder final {
  57  public:
  58   explicit VpxEncoder(const OnEncodedVideoCB& on_encoded_video_callback);
  59   ~VpxEncoder();
  60
  61   void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
  62                         base::TimeTicks capture_timestamp);
  63
  64  private:
  65   void EncodeOnEncodingThread(const scoped_refptr<VideoFrame>& frame,
  66                               base::TimeTicks capture_timestamp);
  67
  68   void ConfigureVp8Encoding(const gfx::Size& size);
  69
  70   // Returns true if |codec_config_| has been filled in at least once.
  71   bool IsInitialized() const;
  72
  73   // Estimate the frame duration from |frame| and |last_frame_timestamp_|.
  74   base::TimeDelta CalculateFrameDuration(
  75       const scoped_refptr<VideoFrame>& frame);
  76
  77   // Used to check that we are destroyed on the same thread we were created.
  78   base::ThreadChecker main_render_thread_checker_;
  79
  80   // Task runner where frames to encode and reply callbacks must happen.
  81   scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
  82
  83   // This callback should be exercised on IO thread.
  84   const OnEncodedVideoCB on_encoded_video_callback_;
  85
  86   // Thread for encoding. Active as long as VpxEncoder exists. All variables
  87   // below this are used in this thread.
  88   base::Thread encoding_thread_;
  89   // VP8 internal objects: configuration, encoder and Vpx Image wrapper.
  90   vpx_codec_enc_cfg_t codec_config_;
  91   vpx_codec_ctx_t encoder_;
  92
  93   // The |VideoFrame::timestamp()| of the last encoded frame.  This is used to
  94   // predict the duration of the next frame.
  95   base::TimeDelta last_frame_timestamp_;
  96
  97   DISALLOW_COPY_AND_ASSIGN(VpxEncoder);
  98 };
  99
 100 VideoTrackRecorder::VpxEncoder::VpxEncoder(
 101     const OnEncodedVideoCB& on_encoded_video_callback)
 102     : on_encoded_video_callback_(on_encoded_video_callback),
 103       encoding_thread_("EncodingThread") {
 104   DCHECK(!on_encoded_video_callback_.is_null());
 105
 106   codec_config_.g_timebase.den = 0;  // Not initialized.
 107
 108   DCHECK(!encoding_thread_.IsRunning());
 109   encoding_thread_.Start();
 110 }
 111
 112 VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
 113   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 114   DCHECK(encoding_thread_.IsRunning());
 115   encoding_thread_.Stop();
 116   vpx_codec_destroy(&encoder_);
 117 }
 118
 119 void VideoTrackRecorder::VpxEncoder::StartFrameEncode(
 120     const scoped_refptr<VideoFrame>& frame,
 121     base::TimeTicks capture_timestamp) {
 122   // Cache the thread sending frames on first frame arrival.
 123   if (!origin_task_runner_.get())
 124     origin_task_runner_ = base::MessageLoop::current()->task_runner();
 125   DCHECK(origin_task_runner_->BelongsToCurrentThread());
 126
 127   encoding_thread_.task_runner()->PostTask(
 128       FROM_HERE, base::Bind(&VpxEncoder::EncodeOnEncodingThread,
 129                             base::Unretained(this), frame, capture_timestamp));
 130 }
 131
 132 void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread(
 133     const scoped_refptr<VideoFrame>& frame,
 134     base::TimeTicks capture_timestamp) {
 135   TRACE_EVENT0("video",
 136                "VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread");
 137   DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
 138
 139   const gfx::Size frame_size = frame->visible_rect().size();
 140   if (!IsInitialized() ||
 141       gfx::Size(codec_config_.g_w, codec_config_.g_h) != frame_size) {
 142     ConfigureVp8Encoding(frame_size);
 143   }
 144
 145   vpx_image_t vpx_image;
 146   vpx_image_t* const result = vpx_img_wrap(&vpx_image,
 147                                            VPX_IMG_FMT_I420,
 148                                            frame_size.width(),
 149                                            frame_size.height(),
 150                                            1  /* align */,
 151                                            frame->data(VideoFrame::kYPlane));
 152   DCHECK_EQ(result, &vpx_image);
 153   vpx_image.planes[VPX_PLANE_Y] = frame->visible_data(VideoFrame::kYPlane);
 154   vpx_image.planes[VPX_PLANE_U] = frame->visible_data(VideoFrame::kUPlane);
 155   vpx_image.planes[VPX_PLANE_V] = frame->visible_data(VideoFrame::kVPlane);
 156   vpx_image.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
 157   vpx_image.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
 158   vpx_image.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
 159
 160   const base::TimeDelta duration = CalculateFrameDuration(frame);
 161   // Encode the frame.  The presentation time stamp argument here is fixed to
 162   // zero to force the encoder to base its single-frame bandwidth calculations
 163   // entirely on |predicted_frame_duration|.
 164   const vpx_codec_err_t ret = vpx_codec_encode(&encoder_,
 165                                                &vpx_image,
 166                                                0  /* pts */,
 167                                                duration.InMicroseconds(),
 168                                                kNoFlags,
 169                                                VPX_DL_REALTIME);
 170   DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
 171                                << vpx_codec_error(&encoder_) << " -"
 172                                << vpx_codec_error_detail(&encoder_);
 173
 174   scoped_ptr<std::string> data(new std::string);
 175   bool keyframe = false;
 176   vpx_codec_iter_t iter = NULL;
 177   const vpx_codec_cx_pkt_t* pkt = NULL;
 178   while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) {
 179     if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
 180       continue;
 181     data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
 182     keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
 183     break;
 184   }
 185   origin_task_runner_->PostTask(FROM_HERE,
 186       base::Bind(&OnFrameEncodeCompleted,
 187                  on_encoded_video_callback_,
 188                  frame,
 189                  base::Passed(&data),
 190                  capture_timestamp,
 191                  keyframe));
 192 }
 193
 194 void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding(
 195     const gfx::Size& size) {
 196   if (IsInitialized()) {
 197     // TODO(mcasas) VP8 quirk/optimisation: If the new |size| is strictly less-
 198     // than-or-equal than the old size, in terms of area, the existing encoder
 199     // instance could be reused after changing |codec_config_.{g_w,g_h}|.
 200     DVLOG(1) << "Destroying/Re-Creating encoder for new frame size: "
 201              << gfx::Size(codec_config_.g_w, codec_config_.g_h).ToString()
 202              << " --> " << size.ToString();
 203     vpx_codec_destroy(&encoder_);
 204   }
 205
 206   const vpx_codec_iface_t* interface = vpx_codec_vp8_cx();
 207   const vpx_codec_err_t result = vpx_codec_enc_config_default(interface,
 208                                                               &codec_config_,
 209                                                               0 /* reserved */);
 210   DCHECK_EQ(VPX_CODEC_OK, result);
 211
 212   // Adjust default bit rate to account for the actual size.
 213   DCHECK_EQ(320u, codec_config_.g_w);
 214   DCHECK_EQ(240u, codec_config_.g_h);
 215   DCHECK_EQ(256u, codec_config_.rc_target_bitrate);
 216   codec_config_.rc_target_bitrate = size.GetArea() *
 217      codec_config_.rc_target_bitrate / codec_config_.g_w / codec_config_.g_h;
 218
 219   DCHECK(size.width());
 220   DCHECK(size.height());
 221   codec_config_.g_w = size.width();
 222   codec_config_.g_h = size.height();
 223   codec_config_.g_pass = VPX_RC_ONE_PASS;
 224
 225   // Timebase is the smallest interval used by the stream, can be set to the
 226   // frame rate or to e.g. microseconds.
 227   codec_config_.g_timebase.num = 1;
 228   codec_config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
 229
 230   // Let the encoder decide where to place the Keyframes, between min and max.
 231   // In VPX_KF_AUTO mode libvpx will sometimes emit keyframes regardless of min/
 232   // max distance out of necessity.
 233   // Note that due to http://crbug.com/440223, it might be necessary to force a
 234   // key frame after 10,000frames since decoding fails after 30,000 non-key
 235   // frames.
 236   codec_config_.kf_mode = VPX_KF_AUTO;
 237   codec_config_.kf_min_dist = 0;
 238   codec_config_.kf_max_dist = 30000;
 239
 240   // Do not saturate CPU utilization just for encoding. On a lower-end system
 241   // with only 1 or 2 cores, use only one thread for encoding. On systems with
 242   // more cores, allow half of the cores to be used for encoding.
 243   codec_config_.g_threads =
 244       std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2);
 245
 246   // Number of frames to consume before producing output.
 247   codec_config_.g_lag_in_frames = 0;
 248
 249   const vpx_codec_err_t ret = vpx_codec_enc_init(&encoder_, interface,
 250                                                  &codec_config_, kNoFlags);
 251   DCHECK_EQ(VPX_CODEC_OK, ret);
 252 }
 253
 254 bool VideoTrackRecorder::VpxEncoder::IsInitialized() const {
 255   DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
 256   return codec_config_.g_timebase.den != 0;
 257 }
 258
 259 base::TimeDelta VideoTrackRecorder::VpxEncoder::CalculateFrameDuration(
 260     const scoped_refptr<VideoFrame>& frame) {
 261   DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
 262
 263   using base::TimeDelta;
 264   TimeDelta predicted_frame_duration;
 265   if (!frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION,
 266                                        &predicted_frame_duration) ||
 267       predicted_frame_duration <= TimeDelta()) {
 268     // The source of the video frame did not provide the frame duration.  Use
 269     // the actual amount of time between the current and previous frame as a
 270     // prediction for the next frame's duration.
 271     // TODO(mcasas): This duration estimation could lead to artifacts if the
 272     // cadence of the received stream is compromised (e.g. camera freeze, pause,
 273     // remote packet loss).  Investigate using GetFrameRate() in this case.
 274     predicted_frame_duration = frame->timestamp() - last_frame_timestamp_;
 275   }
 276   last_frame_timestamp_ = frame->timestamp();
 277   // Make sure |predicted_frame_duration| is in a safe range of values.
 278   const TimeDelta kMaxFrameDuration = TimeDelta::FromSecondsD(1.0 / 8);
 279   const TimeDelta kMinFrameDuration = TimeDelta::FromMilliseconds(1);
 280   return std::min(kMaxFrameDuration, std::max(predicted_frame_duration,
 281                                               kMinFrameDuration));
 282 }
 283
 284 VideoTrackRecorder::VideoTrackRecorder(
 285     const blink::WebMediaStreamTrack& track,
 286     const OnEncodedVideoCB& on_encoded_video_callback)
 287     : track_(track),
 288       encoder_(new VpxEncoder(on_encoded_video_callback)),
 289       weak_factory_(this) {
 290   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 291   DCHECK(!track_.isNull());
 292   DCHECK(track.extraData());
 293   AddToVideoTrack(this,
 294                   media::BindToCurrentLoop(
 295                       base::Bind(&VideoTrackRecorder::OnVideoFrame,
 296                                  weak_factory_.GetWeakPtr())),
 297                   track_);
 298 }
 299
 300 VideoTrackRecorder::~VideoTrackRecorder() {
 301   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 302   RemoveFromVideoTrack(this, track_);
 303   weak_factory_.InvalidateWeakPtrs();
 304   track_.reset();
 305 }
 306
 307 void VideoTrackRecorder::OnVideoFrame(const scoped_refptr<VideoFrame>& frame,
 308                                       base::TimeTicks timestamp) {
 309   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 310   encoder_->StartFrameEncode(frame, timestamp);
 311 }
 312
 313 }  // namespace content