media/filters/vpx_video_decoder.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/filters/vpx_video_decoder.h"
   6
   7 #include <algorithm>
   8 #include <string>
   9 #include <vector>
  10
  11 #include "base/bind.h"
  12 #include "base/callback_helpers.h"
  13 #include "base/command_line.h"
  14 #include "base/location.h"
  15 #include "base/logging.h"
  16 #include "base/single_thread_task_runner.h"
  17 #include "base/stl_util.h"
  18 #include "base/strings/string_number_conversions.h"
  19 #include "base/sys_byteorder.h"
  20 #include "base/sys_info.h"
  21 #include "base/trace_event/trace_event.h"
  22 #include "media/base/bind_to_current_loop.h"
  23 #include "media/base/decoder_buffer.h"
  24 #include "media/base/limits.h"
  25 #include "media/base/media_switches.h"
  26 #include "media/base/pipeline.h"
  27 #include "media/base/video_util.h"
  28
  29 // Include libvpx header files.
  30 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
  31 // backwards compatibility for legacy applications using the library.
  32 #define VPX_CODEC_DISABLE_COMPAT 1
  33 extern "C" {
  34 #include "third_party/libvpx/source/libvpx/vpx/vpx_decoder.h"
  35 #include "third_party/libvpx/source/libvpx/vpx/vpx_frame_buffer.h"
  36 #include "third_party/libvpx/source/libvpx/vpx/vp8dx.h"
  37 }
  38
  39 namespace media {
  40
  41 // Always try to use three threads for video decoding.  There is little reason
  42 // not to since current day CPUs tend to be multi-core and we measured
  43 // performance benefits on older machines such as P4s with hyperthreading.
  44 static const int kDecodeThreads = 2;
  45 static const int kMaxDecodeThreads = 16;
  46
  47 // Returns the number of threads.
  48 static int GetThreadCount(const VideoDecoderConfig& config) {
  49   // Refer to http://crbug.com/93932 for tsan suppressions on decoding.
  50   int decode_threads = kDecodeThreads;
  51
  52   const base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
  53   std::string threads(cmd_line->GetSwitchValueASCII(switches::kVideoThreads));
  54   if (threads.empty() || !base::StringToInt(threads, &decode_threads)) {
  55     if (config.codec() == kCodecVP9) {
  56       // For VP9 decode when using the default thread count, increase the number
  57       // of decode threads to equal the maximum number of tiles possible for
  58       // higher resolution streams.
  59       if (config.coded_size().width() >= 2048)
  60         decode_threads = 8;
  61       else if (config.coded_size().width() >= 1024)
  62         decode_threads = 4;
  63     }
  64
  65     decode_threads = std::min(decode_threads,
  66                               base::SysInfo::NumberOfProcessors());
  67     return decode_threads;
  68   }
  69
  70   decode_threads = std::max(decode_threads, 0);
  71   decode_threads = std::min(decode_threads, kMaxDecodeThreads);
  72   return decode_threads;
  73 }
  74
  75 class VpxVideoDecoder::MemoryPool
  76     : public base::RefCountedThreadSafe<VpxVideoDecoder::MemoryPool> {
  77  public:
  78   MemoryPool();
  79
  80   // Callback that will be called by libvpx when it needs a frame buffer.
  81   // Parameters:
  82   // |user_priv|  Private data passed to libvpx (pointer to memory pool).
  83   // |min_size|   Minimum size needed by libvpx to decompress the next frame.
  84   // |fb|         Pointer to the frame buffer to update.
  85   // Returns 0 on success. Returns < 0 on failure.
  86   static int32 GetVP9FrameBuffer(void* user_priv, size_t min_size,
  87                                  vpx_codec_frame_buffer* fb);
  88
  89   // Callback that will be called by libvpx when the frame buffer is no longer
  90   // being used by libvpx. Parameters:
  91   // |user_priv|  Private data passed to libvpx (pointer to memory pool).
  92   // |fb|         Pointer to the frame buffer that's being released.
  93   static int32 ReleaseVP9FrameBuffer(void *user_priv,
  94                                      vpx_codec_frame_buffer *fb);
  95
  96   // Generates a "no_longer_needed" closure that holds a reference
  97   // to this pool.
  98   base::Closure CreateFrameCallback(void* fb_priv_data);
  99
 100  private:
 101   friend class base::RefCountedThreadSafe<VpxVideoDecoder::MemoryPool>;
 102   ~MemoryPool();
 103
 104   // Reference counted frame buffers used for VP9 decoding. Reference counting
 105   // is done manually because both chromium and libvpx has to release this
 106   // before a buffer can be re-used.
 107   struct VP9FrameBuffer {
 108     VP9FrameBuffer() : ref_cnt(0) {}
 109     std::vector<uint8> data;
 110     uint32 ref_cnt;
 111   };
 112
 113   // Gets the next available frame buffer for use by libvpx.
 114   VP9FrameBuffer* GetFreeFrameBuffer(size_t min_size);
 115
 116   // Method that gets called when a VideoFrame that references this pool gets
 117   // destroyed.
 118   void OnVideoFrameDestroyed(VP9FrameBuffer* frame_buffer);
 119
 120   // Frame buffers to be used by libvpx for VP9 Decoding.
 121   std::vector<VP9FrameBuffer*> frame_buffers_;
 122
 123   DISALLOW_COPY_AND_ASSIGN(MemoryPool);
 124 };
 125
 126 VpxVideoDecoder::MemoryPool::MemoryPool() {}
 127
 128 VpxVideoDecoder::MemoryPool::~MemoryPool() {
 129   STLDeleteElements(&frame_buffers_);
 130 }
 131
 132 VpxVideoDecoder::MemoryPool::VP9FrameBuffer*
 133     VpxVideoDecoder::MemoryPool::GetFreeFrameBuffer(size_t min_size) {
 134   // Check if a free frame buffer exists.
 135   size_t i = 0;
 136   for (; i < frame_buffers_.size(); ++i) {
 137     if (frame_buffers_[i]->ref_cnt == 0)
 138       break;
 139   }
 140
 141   if (i == frame_buffers_.size()) {
 142     // Create a new frame buffer.
 143     frame_buffers_.push_back(new VP9FrameBuffer());
 144   }
 145
 146   // Resize the frame buffer if necessary.
 147   if (frame_buffers_[i]->data.size() < min_size)
 148     frame_buffers_[i]->data.resize(min_size);
 149   return frame_buffers_[i];
 150 }
 151
 152 int32 VpxVideoDecoder::MemoryPool::GetVP9FrameBuffer(
 153     void* user_priv, size_t min_size, vpx_codec_frame_buffer* fb) {
 154   DCHECK(user_priv);
 155   DCHECK(fb);
 156
 157   VpxVideoDecoder::MemoryPool* memory_pool =
 158       static_cast<VpxVideoDecoder::MemoryPool*>(user_priv);
 159
 160   VP9FrameBuffer* fb_to_use = memory_pool->GetFreeFrameBuffer(min_size);
 161   if (fb_to_use == NULL)
 162     return -1;
 163
 164   fb->data = &fb_to_use->data[0];
 165   fb->size = fb_to_use->data.size();
 166   ++fb_to_use->ref_cnt;
 167
 168   // Set the frame buffer's private data to point at the external frame buffer.
 169   fb->priv = static_cast<void*>(fb_to_use);
 170   return 0;
 171 }
 172
 173 int32 VpxVideoDecoder::MemoryPool::ReleaseVP9FrameBuffer(
 174     void *user_priv, vpx_codec_frame_buffer *fb) {
 175   VP9FrameBuffer* frame_buffer = static_cast<VP9FrameBuffer*>(fb->priv);
 176   --frame_buffer->ref_cnt;
 177   return 0;
 178 }
 179
 180 base::Closure VpxVideoDecoder::MemoryPool::CreateFrameCallback(
 181     void* fb_priv_data) {
 182   VP9FrameBuffer* frame_buffer = static_cast<VP9FrameBuffer*>(fb_priv_data);
 183   ++frame_buffer->ref_cnt;
 184   return BindToCurrentLoop(
 185              base::Bind(&MemoryPool::OnVideoFrameDestroyed, this,
 186                         frame_buffer));
 187 }
 188
 189 void VpxVideoDecoder::MemoryPool::OnVideoFrameDestroyed(
 190     VP9FrameBuffer* frame_buffer) {
 191   --frame_buffer->ref_cnt;
 192 }
 193
 194 VpxVideoDecoder::VpxVideoDecoder(
 195     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner)
 196     : task_runner_(task_runner),
 197       state_(kUninitialized),
 198       vpx_codec_(NULL),
 199       vpx_codec_alpha_(NULL) {}
 200
 201 VpxVideoDecoder::~VpxVideoDecoder() {
 202   DCHECK(task_runner_->BelongsToCurrentThread());
 203   CloseDecoder();
 204 }
 205
 206 std::string VpxVideoDecoder::GetDisplayName() const {
 207   return "VpxVideoDecoder";
 208 }
 209
 210 void VpxVideoDecoder::Initialize(const VideoDecoderConfig& config,
 211                                  bool low_delay,
 212                                  const InitCB& init_cb,
 213                                  const OutputCB& output_cb) {
 214   DCHECK(task_runner_->BelongsToCurrentThread());
 215   DCHECK(config.IsValidConfig());
 216   DCHECK(!config.is_encrypted());
 217   DCHECK(decode_cb_.is_null());
 218
 219   InitCB bound_init_cb = BindToCurrentLoop(init_cb);
 220
 221   if (!ConfigureDecoder(config)) {
 222     bound_init_cb.Run(false);
 223     return;
 224   }
 225
 226   // Success!
 227   config_ = config;
 228   state_ = kNormal;
 229   output_cb_ = BindToCurrentLoop(output_cb);
 230   bound_init_cb.Run(true);
 231 }
 232
 233 static vpx_codec_ctx* InitializeVpxContext(vpx_codec_ctx* context,
 234                                            const VideoDecoderConfig& config) {
 235   context = new vpx_codec_ctx();
 236   vpx_codec_dec_cfg_t vpx_config = {0};
 237   vpx_config.w = config.coded_size().width();
 238   vpx_config.h = config.coded_size().height();
 239   vpx_config.threads = GetThreadCount(config);
 240
 241   vpx_codec_err_t status = vpx_codec_dec_init(context,
 242                                               config.codec() == kCodecVP9 ?
 243                                                   vpx_codec_vp9_dx() :
 244                                                   vpx_codec_vp8_dx(),
 245                                               &vpx_config,
 246                                               0);
 247   if (status != VPX_CODEC_OK) {
 248     LOG(ERROR) << "vpx_codec_dec_init failed, status=" << status;
 249     delete context;
 250     return NULL;
 251   }
 252   return context;
 253 }
 254
 255 bool VpxVideoDecoder::ConfigureDecoder(const VideoDecoderConfig& config) {
 256   if (config.codec() != kCodecVP8 && config.codec() != kCodecVP9)
 257     return false;
 258
 259   // In VP8 videos, only those with alpha are handled by VpxVideoDecoder. All
 260   // other VP8 videos go to FFmpegVideoDecoder.
 261   if (config.codec() == kCodecVP8 && config.format() != PIXEL_FORMAT_YV12A)
 262     return false;
 263
 264   CloseDecoder();
 265
 266   vpx_codec_ = InitializeVpxContext(vpx_codec_, config);
 267   if (!vpx_codec_)
 268     return false;
 269
 270   // We use our own buffers for VP9 so that there is no need to copy data after
 271   // decoding.
 272   if (config.codec() == kCodecVP9) {
 273     memory_pool_ = new MemoryPool();
 274     if (vpx_codec_set_frame_buffer_functions(vpx_codec_,
 275                                              &MemoryPool::GetVP9FrameBuffer,
 276                                              &MemoryPool::ReleaseVP9FrameBuffer,
 277                                              memory_pool_.get())) {
 278       LOG(ERROR) << "Failed to configure external buffers.";
 279       return false;
 280     }
 281   }
 282
 283   if (config.format() == PIXEL_FORMAT_YV12A) {
 284     vpx_codec_alpha_ = InitializeVpxContext(vpx_codec_alpha_, config);
 285     if (!vpx_codec_alpha_)
 286       return false;
 287   }
 288
 289   return true;
 290 }
 291
 292 void VpxVideoDecoder::CloseDecoder() {
 293   if (vpx_codec_) {
 294     vpx_codec_destroy(vpx_codec_);
 295     delete vpx_codec_;
 296     vpx_codec_ = NULL;
 297     memory_pool_ = NULL;
 298   }
 299   if (vpx_codec_alpha_) {
 300     vpx_codec_destroy(vpx_codec_alpha_);
 301     delete vpx_codec_alpha_;
 302     vpx_codec_alpha_ = NULL;
 303   }
 304 }
 305
 306 void VpxVideoDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
 307                              const DecodeCB& decode_cb) {
 308   DCHECK(task_runner_->BelongsToCurrentThread());
 309   DCHECK(!decode_cb.is_null());
 310   CHECK_NE(state_, kUninitialized);
 311   CHECK(decode_cb_.is_null()) << "Overlapping decodes are not supported.";
 312
 313   decode_cb_ = BindToCurrentLoop(decode_cb);
 314
 315   if (state_ == kError) {
 316     base::ResetAndReturn(&decode_cb_).Run(kDecodeError);
 317     return;
 318   }
 319
 320   // Return empty frames if decoding has finished.
 321   if (state_ == kDecodeFinished) {
 322     base::ResetAndReturn(&decode_cb_).Run(kOk);
 323     return;
 324   }
 325
 326   DecodeBuffer(buffer);
 327 }
 328
 329 void VpxVideoDecoder::Reset(const base::Closure& closure) {
 330   DCHECK(task_runner_->BelongsToCurrentThread());
 331   DCHECK(decode_cb_.is_null());
 332
 333   state_ = kNormal;
 334   task_runner_->PostTask(FROM_HERE, closure);
 335 }
 336
 337 void VpxVideoDecoder::DecodeBuffer(const scoped_refptr<DecoderBuffer>& buffer) {
 338   DCHECK(task_runner_->BelongsToCurrentThread());
 339   DCHECK_NE(state_, kUninitialized);
 340   DCHECK_NE(state_, kDecodeFinished);
 341   DCHECK_NE(state_, kError);
 342   DCHECK(!decode_cb_.is_null());
 343   DCHECK(buffer.get());
 344
 345   // Transition to kDecodeFinished on the first end of stream buffer.
 346   if (state_ == kNormal && buffer->end_of_stream()) {
 347     state_ = kDecodeFinished;
 348     base::ResetAndReturn(&decode_cb_).Run(kOk);
 349     return;
 350   }
 351
 352   scoped_refptr<VideoFrame> video_frame;
 353   if (!VpxDecode(buffer, &video_frame)) {
 354     state_ = kError;
 355     base::ResetAndReturn(&decode_cb_).Run(kDecodeError);
 356     return;
 357   }
 358
 359   if (video_frame.get())
 360     output_cb_.Run(video_frame);
 361
 362   // VideoDecoderShim expects that |decode_cb| is called only after
 363   // |output_cb_|.
 364   base::ResetAndReturn(&decode_cb_).Run(kOk);
 365 }
 366
 367 bool VpxVideoDecoder::VpxDecode(const scoped_refptr<DecoderBuffer>& buffer,
 368                                 scoped_refptr<VideoFrame>* video_frame) {
 369   DCHECK(video_frame);
 370   DCHECK(!buffer->end_of_stream());
 371
 372   // Pass |buffer| to libvpx.
 373   int64 timestamp = buffer->timestamp().InMicroseconds();
 374   void* user_priv = reinterpret_cast<void*>(&timestamp);
 375
 376   {
 377     TRACE_EVENT1("video", "vpx_codec_decode", "timestamp", timestamp);
 378     vpx_codec_err_t status = vpx_codec_decode(vpx_codec_,
 379                                               buffer->data(),
 380                                               buffer->data_size(),
 381                                               user_priv,
 382                                               0);
 383     if (status != VPX_CODEC_OK) {
 384       LOG(ERROR) << "vpx_codec_decode() failed, status=" << status;
 385       return false;
 386     }
 387   }
 388
 389   // Gets pointer to decoded data.
 390   vpx_codec_iter_t iter = NULL;
 391   const vpx_image_t* vpx_image = vpx_codec_get_frame(vpx_codec_, &iter);
 392   if (!vpx_image) {
 393     *video_frame = NULL;
 394     return true;
 395   }
 396
 397   if (vpx_image->user_priv != reinterpret_cast<void*>(&timestamp)) {
 398     LOG(ERROR) << "Invalid output timestamp.";
 399     return false;
 400   }
 401
 402   const vpx_image_t* vpx_image_alpha = NULL;
 403   if (vpx_codec_alpha_ && buffer->side_data_size() >= 8) {
 404     // Pass alpha data to libvpx.
 405     int64 timestamp_alpha = buffer->timestamp().InMicroseconds();
 406     void* user_priv_alpha = reinterpret_cast<void*>(&timestamp_alpha);
 407
 408     // First 8 bytes of side data is side_data_id in big endian.
 409     const uint64 side_data_id = base::NetToHost64(
 410         *(reinterpret_cast<const uint64*>(buffer->side_data())));
 411     if (side_data_id == 1) {
 412       {
 413         TRACE_EVENT1("video", "vpx_codec_decode_alpha",
 414                      "timestamp_alpha", timestamp_alpha);
 415         vpx_codec_err_t status = vpx_codec_decode(vpx_codec_alpha_,
 416                                                   buffer->side_data() + 8,
 417                                                   buffer->side_data_size() - 8,
 418                                                   user_priv_alpha,
 419                                                   0);
 420         if (status != VPX_CODEC_OK) {
 421           LOG(ERROR) << "vpx_codec_decode() failed on alpha, status=" << status;
 422           return false;
 423         }
 424       }
 425
 426       // Gets pointer to decoded data.
 427       vpx_codec_iter_t iter_alpha = NULL;
 428       vpx_image_alpha = vpx_codec_get_frame(vpx_codec_alpha_, &iter_alpha);
 429       if (!vpx_image_alpha) {
 430         *video_frame = NULL;
 431         return true;
 432       }
 433
 434       if (vpx_image_alpha->user_priv !=
 435           reinterpret_cast<void*>(&timestamp_alpha)) {
 436         LOG(ERROR) << "Invalid output timestamp on alpha.";
 437         return false;
 438       }
 439
 440       if (vpx_image_alpha->d_h != vpx_image->d_h ||
 441           vpx_image_alpha->d_w != vpx_image->d_w) {
 442         LOG(ERROR) << "The alpha plane dimensions are not the same as the "
 443                       "image dimensions.";
 444         return false;
 445       }
 446     }
 447   }
 448
 449   CopyVpxImageTo(vpx_image, vpx_image_alpha, video_frame);
 450   (*video_frame)->set_timestamp(base::TimeDelta::FromMicroseconds(timestamp));
 451   return true;
 452 }
 453
 454 void VpxVideoDecoder::CopyVpxImageTo(const vpx_image* vpx_image,
 455                                      const struct vpx_image* vpx_image_alpha,
 456                                      scoped_refptr<VideoFrame>* video_frame) {
 457   CHECK(vpx_image);
 458   CHECK(vpx_image->fmt == VPX_IMG_FMT_I420 ||
 459         vpx_image->fmt == VPX_IMG_FMT_YV12 ||
 460         vpx_image->fmt == VPX_IMG_FMT_I444);
 461
 462   VideoPixelFormat codec_format = PIXEL_FORMAT_YV12;
 463   int uv_rows = (vpx_image->d_h + 1) / 2;
 464
 465   if (vpx_image->fmt == VPX_IMG_FMT_I444) {
 466     CHECK(!vpx_codec_alpha_);
 467     codec_format = PIXEL_FORMAT_YV24;
 468     uv_rows = vpx_image->d_h;
 469   } else if (vpx_codec_alpha_) {
 470     codec_format = PIXEL_FORMAT_YV12A;
 471   }
 472
 473   // Default to the color space from the config, but if the bistream specifies
 474   // one, prefer that instead.
 475   ColorSpace color_space = config_.color_space();
 476   if (vpx_image->cs == VPX_CS_BT_709)
 477     color_space = COLOR_SPACE_HD_REC709;
 478   else if (vpx_image->cs == VPX_CS_BT_601)
 479     color_space = COLOR_SPACE_SD_REC601;
 480
 481   // The mixed |w|/|d_h| in |coded_size| is intentional. Setting the correct
 482   // coded width is necessary to allow coalesced memory access, which may avoid
 483   // frame copies. Setting the correct coded height however does not have any
 484   // benefit, and only risk copying too much data.
 485   const gfx::Size coded_size(vpx_image->w, vpx_image->d_h);
 486   const gfx::Size visible_size(vpx_image->d_w, vpx_image->d_h);
 487
 488   if (!vpx_codec_alpha_ && memory_pool_.get()) {
 489     *video_frame = VideoFrame::WrapExternalYuvData(
 490         codec_format,
 491         coded_size, gfx::Rect(visible_size), config_.natural_size(),
 492         vpx_image->stride[VPX_PLANE_Y],
 493         vpx_image->stride[VPX_PLANE_U],
 494         vpx_image->stride[VPX_PLANE_V],
 495         vpx_image->planes[VPX_PLANE_Y],
 496         vpx_image->planes[VPX_PLANE_U],
 497         vpx_image->planes[VPX_PLANE_V],
 498         kNoTimestamp());
 499     video_frame->get()->AddDestructionObserver(
 500         memory_pool_->CreateFrameCallback(vpx_image->fb_priv));
 501     video_frame->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE,
 502                                                color_space);
 503     return;
 504   }
 505
 506   *video_frame = frame_pool_.CreateFrame(
 507       codec_format,
 508       visible_size,
 509       gfx::Rect(visible_size),
 510       config_.natural_size(),
 511       kNoTimestamp());
 512   video_frame->get()->metadata()->SetInteger(VideoFrameMetadata::COLOR_SPACE,
 513                                              color_space);
 514
 515   CopyYPlane(vpx_image->planes[VPX_PLANE_Y],
 516              vpx_image->stride[VPX_PLANE_Y],
 517              vpx_image->d_h,
 518              video_frame->get());
 519   CopyUPlane(vpx_image->planes[VPX_PLANE_U],
 520              vpx_image->stride[VPX_PLANE_U],
 521              uv_rows,
 522              video_frame->get());
 523   CopyVPlane(vpx_image->planes[VPX_PLANE_V],
 524              vpx_image->stride[VPX_PLANE_V],
 525              uv_rows,
 526              video_frame->get());
 527   if (!vpx_codec_alpha_)
 528     return;
 529   if (!vpx_image_alpha) {
 530     MakeOpaqueAPlane(
 531         vpx_image->stride[VPX_PLANE_Y], vpx_image->d_h, video_frame->get());
 532     return;
 533   }
 534   CopyAPlane(vpx_image_alpha->planes[VPX_PLANE_Y],
 535              vpx_image_alpha->stride[VPX_PLANE_Y],
 536              vpx_image_alpha->d_h,
 537              video_frame->get());
 538 }
 539
 540 }  // namespace media