content/renderer/media/video_track_recorder.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "content/renderer/media/video_track_recorder.h"
   6
   7 #include "base/bind.h"
   8 #include "base/logging.h"
   9 #include "base/sys_info.h"
  10 #include "base/threading/thread.h"
  11 #include "base/time/time.h"
  12 #include "base/trace_event/trace_event.h"
  13 #include "media/base/video_frame.h"
  14
  15 extern "C" {
  16 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
  17 // backwards compatibility for legacy applications using the library.
  18 #define VPX_CODEC_DISABLE_COMPAT 1
  19 #include "third_party/libvpx_new/source/libvpx/vpx/vp8cx.h"
  20 #include "third_party/libvpx_new/source/libvpx/vpx/vpx_encoder.h"
  21 }
  22
  23 using media::VideoFrame;
  24 using media::VideoFrameMetadata;
  25
  26 namespace content {
  27
  28 namespace {
  29
  30 const vpx_codec_flags_t kNoFlags = 0;
  31
  32 // Originally from remoting/codec/scoped_vpx_codec.h.
  33 // TODO(mcasas): Refactor into a common location.
  34 struct VpxCodecDeleter {
  35   void operator()(vpx_codec_ctx_t* codec) {
  36     if (!codec)
  37       return;
  38     vpx_codec_err_t ret = vpx_codec_destroy(codec);
  39     CHECK_EQ(ret, VPX_CODEC_OK);
  40     delete codec;
  41   }
  42 };
  43
  44 typedef scoped_ptr<vpx_codec_ctx_t, VpxCodecDeleter> ScopedVpxCodecCtxPtr;
  45
  46 void OnFrameEncodeCompleted(
  47     const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
  48     const scoped_refptr<VideoFrame>& frame,
  49     scoped_ptr<std::string> data,
  50     base::TimeTicks capture_timestamp,
  51     bool keyframe) {
  52   DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "<< data->length() << "B, "
  53            << capture_timestamp << " ms";
  54   on_encoded_video_cb.Run(frame, base::StringPiece(*data), capture_timestamp,
  55                           keyframe);
  56 }
  57
  58 }  // anonymous namespace
  59
  60 // Inner class encapsulating all libvpx interactions and the encoding+delivery
  61 // of received frames. Limitation: Only VP8 is supported for the time being.
  62 // This class must be ref-counted because the MediaStreamVideoTrack will hold a
  63 // reference to it, via the callback MediaStreamVideoSink passes along, and it's
  64 // unknown when exactly it will release that reference. This class:
  65 // - is created and destroyed on its parent's thread (usually the main Render
  66 // thread);
  67 // - receives VideoFrames and Run()s the callbacks on |origin_task_runner_|,
  68 // which is cached on first frame arrival, and is supposed to be the render IO
  69 // thread, but this is not enforced;
  70 // - uses an internal |encoding_thread_| for libvpx interactions, notably for
  71 // encoding (which might take some time).
  72 class VideoTrackRecorder::VpxEncoder final
  73     : public base::RefCountedThreadSafe<VpxEncoder> {
  74  public:
  75   static void ShutdownEncoder(scoped_ptr<base::Thread> encoding_thread,
  76                               ScopedVpxCodecCtxPtr encoder);
  77
  78   explicit VpxEncoder(const OnEncodedVideoCB& on_encoded_video_callback);
  79
  80   void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
  81                         base::TimeTicks capture_timestamp);
  82
  83  private:
  84   friend class base::RefCountedThreadSafe<VpxEncoder>;
  85   ~VpxEncoder();
  86
  87   void EncodeOnEncodingThread(const scoped_refptr<VideoFrame>& frame,
  88                               base::TimeTicks capture_timestamp);
  89
  90   void ConfigureVp8Encoding(const gfx::Size& size);
  91
  92   // Returns true if |codec_config_| has been filled in at least once.
  93   bool IsInitialized() const;
  94
  95   // Estimate the frame duration from |frame| and |last_frame_timestamp_|.
  96   base::TimeDelta CalculateFrameDuration(
  97       const scoped_refptr<VideoFrame>& frame);
  98
  99   // Used to shutdown properly on the same thread we were created.
 100   const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
 101
 102   // Task runner where frames to encode and reply callbacks must happen.
 103   scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
 104
 105   // This callback should be exercised on IO thread.
 106   const OnEncodedVideoCB on_encoded_video_callback_;
 107
 108   // Thread for encoding. Active for the lifetime of VpxEncoder. All variables
 109   // below this are used in this thread.
 110   scoped_ptr<base::Thread> encoding_thread_;
 111   // VP8 internal objects: configuration and encoder.
 112   vpx_codec_enc_cfg_t codec_config_;
 113   // |encoder_| is a special scoped pointer to guarantee proper destruction.
 114   // Again, it should only be accessed on |encoding_thread_|.
 115   ScopedVpxCodecCtxPtr encoder_;
 116
 117   // The |VideoFrame::timestamp()| of the last encoded frame.  This is used to
 118   // predict the duration of the next frame.
 119   base::TimeDelta last_frame_timestamp_;
 120
 121   DISALLOW_COPY_AND_ASSIGN(VpxEncoder);
 122 };
 123
 124 // static
 125 void VideoTrackRecorder::VpxEncoder::ShutdownEncoder(
 126     scoped_ptr<base::Thread> encoding_thread,
 127     ScopedVpxCodecCtxPtr encoder) {
 128   DCHECK(encoding_thread->IsRunning());
 129   encoding_thread->Stop();
 130   // Both |encoding_thread| and |encoder| will be destroyed at end-of-scope.
 131 }
 132
 133 VideoTrackRecorder::VpxEncoder::VpxEncoder(
 134     const OnEncodedVideoCB& on_encoded_video_callback)
 135     : main_task_runner_(base::MessageLoop::current()->task_runner()),
 136       on_encoded_video_callback_(on_encoded_video_callback),
 137       encoding_thread_(new base::Thread("EncodingThread")) {
 138   DCHECK(!on_encoded_video_callback_.is_null());
 139
 140   codec_config_.g_timebase.den = 0;  // Not initialized.
 141
 142   DCHECK(!encoding_thread_->IsRunning());
 143   encoding_thread_->Start();
 144 }
 145
 146 VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
 147   main_task_runner_->PostTask(FROM_HERE,
 148                               base::Bind(&VpxEncoder::ShutdownEncoder,
 149                                          base::Passed(&encoding_thread_),
 150                                          base::Passed(&encoder_)));
 151 }
 152
 153 void VideoTrackRecorder::VpxEncoder::StartFrameEncode(
 154     const scoped_refptr<VideoFrame>& frame,
 155     base::TimeTicks capture_timestamp) {
 156   // Cache the thread sending frames on first frame arrival.
 157   if (!origin_task_runner_.get())
 158     origin_task_runner_ = base::MessageLoop::current()->task_runner();
 159   DCHECK(origin_task_runner_->BelongsToCurrentThread());
 160
 161   encoding_thread_->task_runner()->PostTask(
 162       FROM_HERE, base::Bind(&VpxEncoder::EncodeOnEncodingThread,
 163                             this, frame, capture_timestamp));
 164 }
 165
 166 void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread(
 167     const scoped_refptr<VideoFrame>& frame,
 168     base::TimeTicks capture_timestamp) {
 169   TRACE_EVENT0("video",
 170                "VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread");
 171   DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread());
 172
 173   const gfx::Size frame_size = frame->visible_rect().size();
 174   if (!IsInitialized() ||
 175       gfx::Size(codec_config_.g_w, codec_config_.g_h) != frame_size) {
 176     ConfigureVp8Encoding(frame_size);
 177   }
 178
 179   vpx_image_t vpx_image;
 180   vpx_image_t* const result = vpx_img_wrap(&vpx_image,
 181                                            VPX_IMG_FMT_I420,
 182                                            frame_size.width(),
 183                                            frame_size.height(),
 184                                            1  /* align */,
 185                                            frame->data(VideoFrame::kYPlane));
 186   DCHECK_EQ(result, &vpx_image);
 187   vpx_image.planes[VPX_PLANE_Y] = frame->visible_data(VideoFrame::kYPlane);
 188   vpx_image.planes[VPX_PLANE_U] = frame->visible_data(VideoFrame::kUPlane);
 189   vpx_image.planes[VPX_PLANE_V] = frame->visible_data(VideoFrame::kVPlane);
 190   vpx_image.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
 191   vpx_image.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
 192   vpx_image.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
 193
 194   const base::TimeDelta duration = CalculateFrameDuration(frame);
 195   // Encode the frame.  The presentation time stamp argument here is fixed to
 196   // zero to force the encoder to base its single-frame bandwidth calculations
 197   // entirely on |predicted_frame_duration|.
 198   const vpx_codec_err_t ret = vpx_codec_encode(encoder_.get(),
 199                                                &vpx_image,
 200                                                0  /* pts */,
 201                                                duration.InMicroseconds(),
 202                                                kNoFlags,
 203                                                VPX_DL_REALTIME);
 204   DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
 205                                << vpx_codec_error(encoder_.get()) << " -"
 206                                << vpx_codec_error_detail(encoder_.get());
 207
 208   scoped_ptr<std::string> data(new std::string);
 209   bool keyframe = false;
 210   vpx_codec_iter_t iter = NULL;
 211   const vpx_codec_cx_pkt_t* pkt = NULL;
 212   while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) {
 213     if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
 214       continue;
 215     data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
 216     keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
 217     break;
 218   }
 219   origin_task_runner_->PostTask(FROM_HERE,
 220       base::Bind(OnFrameEncodeCompleted,
 221                  on_encoded_video_callback_,
 222                  frame,
 223                  base::Passed(&data),
 224                  capture_timestamp,
 225                  keyframe));
 226 }
 227
 228 void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding(
 229     const gfx::Size& size) {
 230   if (IsInitialized()) {
 231     // TODO(mcasas) VP8 quirk/optimisation: If the new |size| is strictly less-
 232     // than-or-equal than the old size, in terms of area, the existing encoder
 233     // instance could be reused after changing |codec_config_.{g_w,g_h}|.
 234     DVLOG(1) << "Destroying/Re-Creating encoder for new frame size: "
 235              << gfx::Size(codec_config_.g_w, codec_config_.g_h).ToString()
 236              << " --> " << size.ToString();
 237     encoder_.reset();
 238   }
 239
 240   const vpx_codec_iface_t* interface = vpx_codec_vp8_cx();
 241   const vpx_codec_err_t result = vpx_codec_enc_config_default(interface,
 242                                                               &codec_config_,
 243                                                               0 /* reserved */);
 244   DCHECK_EQ(VPX_CODEC_OK, result);
 245
 246   // Adjust default bit rate to account for the actual size.
 247   DCHECK_EQ(320u, codec_config_.g_w);
 248   DCHECK_EQ(240u, codec_config_.g_h);
 249   DCHECK_EQ(256u, codec_config_.rc_target_bitrate);
 250   codec_config_.rc_target_bitrate = size.GetArea() *
 251      codec_config_.rc_target_bitrate / codec_config_.g_w / codec_config_.g_h;
 252
 253   DCHECK(size.width());
 254   DCHECK(size.height());
 255   codec_config_.g_w = size.width();
 256   codec_config_.g_h = size.height();
 257   codec_config_.g_pass = VPX_RC_ONE_PASS;
 258
 259   // Timebase is the smallest interval used by the stream, can be set to the
 260   // frame rate or to e.g. microseconds.
 261   codec_config_.g_timebase.num = 1;
 262   codec_config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
 263
 264   // Let the encoder decide where to place the Keyframes, between min and max.
 265   // In VPX_KF_AUTO mode libvpx will sometimes emit keyframes regardless of min/
 266   // max distance out of necessity.
 267   // Note that due to http://crbug.com/440223, it might be necessary to force a
 268   // key frame after 10,000frames since decoding fails after 30,000 non-key
 269   // frames.
 270   codec_config_.kf_mode = VPX_KF_AUTO;
 271   codec_config_.kf_min_dist = 0;
 272   codec_config_.kf_max_dist = 30000;
 273
 274   // Do not saturate CPU utilization just for encoding. On a lower-end system
 275   // with only 1 or 2 cores, use only one thread for encoding. On systems with
 276   // more cores, allow half of the cores to be used for encoding.
 277   codec_config_.g_threads =
 278       std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2);
 279
 280   // Number of frames to consume before producing output.
 281   codec_config_.g_lag_in_frames = 0;
 282
 283   DCHECK(!encoder_);
 284   encoder_.reset(new vpx_codec_ctx_t);
 285   const vpx_codec_err_t ret = vpx_codec_enc_init(encoder_.get(), interface,
 286                                                  &codec_config_, kNoFlags);
 287   DCHECK_EQ(VPX_CODEC_OK, ret);
 288 }
 289
 290 bool VideoTrackRecorder::VpxEncoder::IsInitialized() const {
 291   DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread());
 292   return codec_config_.g_timebase.den != 0;
 293 }
 294
 295 base::TimeDelta VideoTrackRecorder::VpxEncoder::CalculateFrameDuration(
 296     const scoped_refptr<VideoFrame>& frame) {
 297   DCHECK(encoding_thread_->task_runner()->BelongsToCurrentThread());
 298
 299   using base::TimeDelta;
 300   TimeDelta predicted_frame_duration;
 301   if (!frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION,
 302                                        &predicted_frame_duration) ||
 303       predicted_frame_duration <= TimeDelta()) {
 304     // The source of the video frame did not provide the frame duration.  Use
 305     // the actual amount of time between the current and previous frame as a
 306     // prediction for the next frame's duration.
 307     // TODO(mcasas): This duration estimation could lead to artifacts if the
 308     // cadence of the received stream is compromised (e.g. camera freeze, pause,
 309     // remote packet loss).  Investigate using GetFrameRate() in this case.
 310     predicted_frame_duration = frame->timestamp() - last_frame_timestamp_;
 311   }
 312   last_frame_timestamp_ = frame->timestamp();
 313   // Make sure |predicted_frame_duration| is in a safe range of values.
 314   const TimeDelta kMaxFrameDuration = TimeDelta::FromSecondsD(1.0 / 8);
 315   const TimeDelta kMinFrameDuration = TimeDelta::FromMilliseconds(1);
 316   return std::min(kMaxFrameDuration, std::max(predicted_frame_duration,
 317                                               kMinFrameDuration));
 318 }
 319
 320 VideoTrackRecorder::VideoTrackRecorder(
 321     const blink::WebMediaStreamTrack& track,
 322     const OnEncodedVideoCB& on_encoded_video_callback)
 323     : track_(track),
 324       encoder_(new VpxEncoder(on_encoded_video_callback)) {
 325   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 326   DCHECK(!track_.isNull());
 327   DCHECK(track_.extraData());
 328
 329   // StartFrameEncode() will be called on Render IO thread.
 330   AddToVideoTrack(this,
 331                   base::Bind(&VideoTrackRecorder::VpxEncoder::StartFrameEncode,
 332                              encoder_),
 333                   track_);
 334 }
 335
 336 VideoTrackRecorder::~VideoTrackRecorder() {
 337   DCHECK(main_render_thread_checker_.CalledOnValidThread());
 338   RemoveFromVideoTrack(this, track_);
 339   track_.reset();
 340 }
 341
 342 void VideoTrackRecorder::OnVideoFrameForTesting(
 343     const scoped_refptr<media::VideoFrame>& frame,
 344     base::TimeTicks timestamp) {
 345   encoder_->StartFrameEncode(frame, timestamp);
 346 }
 347
 348 }  // namespace content