media/base/video_frame.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/base/video_frame.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/callback_helpers.h"
  11 #include "base/logging.h"
  12 #include "base/memory/aligned_memory.h"
  13 #include "base/strings/string_piece.h"
  14 #include "gpu/command_buffer/common/mailbox_holder.h"
  15 #include "media/base/limits.h"
  16 #include "media/base/video_util.h"
  17 #include "third_party/skia/include/core/SkBitmap.h"
  18
  19 namespace media {
  20
  21 // static
  22 scoped_refptr<VideoFrame> VideoFrame::CreateFrame(
  23     VideoFrame::Format format,
  24     const gfx::Size& coded_size,
  25     const gfx::Rect& visible_rect,
  26     const gfx::Size& natural_size,
  27     base::TimeDelta timestamp) {
  28   DCHECK(IsValidConfig(format, coded_size, visible_rect, natural_size));
  29   scoped_refptr<VideoFrame> frame(new VideoFrame(
  30       format, coded_size, visible_rect, natural_size, timestamp, false));
  31   switch (format) {
  32     case VideoFrame::YV12:
  33     case VideoFrame::YV12A:
  34     case VideoFrame::YV16:
  35     case VideoFrame::I420:
  36     case VideoFrame::YV12J:
  37       frame->AllocateYUV();
  38       break;
  39     default:
  40       LOG(FATAL) << "Unsupported frame format: " << format;
  41   }
  42   return frame;
  43 }
  44
  45 // static
  46 std::string VideoFrame::FormatToString(VideoFrame::Format format) {
  47   switch (format) {
  48     case VideoFrame::UNKNOWN:
  49       return "UNKNOWN";
  50     case VideoFrame::YV12:
  51       return "YV12";
  52     case VideoFrame::YV16:
  53       return "YV16";
  54     case VideoFrame::I420:
  55       return "I420";
  56     case VideoFrame::NATIVE_TEXTURE:
  57       return "NATIVE_TEXTURE";
  58 #if defined(VIDEO_HOLE)
  59     case VideoFrame::HOLE:
  60       return "HOLE";
  61 #endif  // defined(VIDEO_HOLE)
  62     case VideoFrame::YV12A:
  63       return "YV12A";
  64     case VideoFrame::YV12J:
  65       return "YV12J";
  66     case VideoFrame::HISTOGRAM_MAX:
  67       return "HISTOGRAM_MAX";
  68   }
  69   NOTREACHED() << "Invalid videoframe format provided: " << format;
  70   return "";
  71 }
  72
  73 // static
  74 bool VideoFrame::IsValidConfig(VideoFrame::Format format,
  75                                const gfx::Size& coded_size,
  76                                const gfx::Rect& visible_rect,
  77                                const gfx::Size& natural_size) {
  78   return (format != VideoFrame::UNKNOWN &&
  79           !coded_size.IsEmpty() &&
  80           coded_size.GetArea() <= limits::kMaxCanvas &&
  81           coded_size.width() <= limits::kMaxDimension &&
  82           coded_size.height() <= limits::kMaxDimension &&
  83           !visible_rect.IsEmpty() &&
  84           visible_rect.x() >= 0 && visible_rect.y() >= 0 &&
  85           visible_rect.right() <= coded_size.width() &&
  86           visible_rect.bottom() <= coded_size.height() &&
  87           !natural_size.IsEmpty() &&
  88           natural_size.GetArea() <= limits::kMaxCanvas &&
  89           natural_size.width() <= limits::kMaxDimension &&
  90           natural_size.height() <= limits::kMaxDimension);
  91 }
  92
  93 // static
  94 scoped_refptr<VideoFrame> VideoFrame::WrapNativeTexture(
  95     scoped_ptr<gpu::MailboxHolder> mailbox_holder,
  96     const ReleaseMailboxCB& mailbox_holder_release_cb,
  97     const gfx::Size& coded_size,
  98     const gfx::Rect& visible_rect,
  99     const gfx::Size& natural_size,
 100     base::TimeDelta timestamp,
 101     const ReadPixelsCB& read_pixels_cb) {
 102   scoped_refptr<VideoFrame> frame(new VideoFrame(NATIVE_TEXTURE,
 103                                                  coded_size,
 104                                                  visible_rect,
 105                                                  natural_size,
 106                                                  timestamp,
 107                                                  false));
 108   frame->mailbox_holder_ = mailbox_holder.Pass();
 109   frame->mailbox_holder_release_cb_ = mailbox_holder_release_cb;
 110   frame->read_pixels_cb_ = read_pixels_cb;
 111
 112   return frame;
 113 }
 114
 115 void VideoFrame::ReadPixelsFromNativeTexture(const SkBitmap& pixels) {
 116   DCHECK_EQ(format_, NATIVE_TEXTURE);
 117   if (!read_pixels_cb_.is_null())
 118     read_pixels_cb_.Run(pixels);
 119 }
 120
 121 // static
 122 scoped_refptr<VideoFrame> VideoFrame::WrapExternalPackedMemory(
 123     Format format,
 124     const gfx::Size& coded_size,
 125     const gfx::Rect& visible_rect,
 126     const gfx::Size& natural_size,
 127     uint8* data,
 128     size_t data_size,
 129     base::SharedMemoryHandle handle,
 130     base::TimeDelta timestamp,
 131     const base::Closure& no_longer_needed_cb) {
 132   if (data_size < AllocationSize(format, coded_size))
 133     return NULL;
 134
 135   switch (format) {
 136     case I420: {
 137       scoped_refptr<VideoFrame> frame(new VideoFrame(
 138           format, coded_size, visible_rect, natural_size, timestamp, false));
 139       frame->shared_memory_handle_ = handle;
 140       frame->strides_[kYPlane] = coded_size.width();
 141       frame->strides_[kUPlane] = coded_size.width() / 2;
 142       frame->strides_[kVPlane] = coded_size.width() / 2;
 143       frame->data_[kYPlane] = data;
 144       frame->data_[kUPlane] = data + coded_size.GetArea();
 145       frame->data_[kVPlane] = data + (coded_size.GetArea() * 5 / 4);
 146       frame->no_longer_needed_cb_ = no_longer_needed_cb;
 147       return frame;
 148     }
 149     default:
 150       NOTIMPLEMENTED();
 151       return NULL;
 152   }
 153 }
 154
 155 // static
 156 scoped_refptr<VideoFrame> VideoFrame::WrapExternalYuvData(
 157     Format format,
 158     const gfx::Size& coded_size,
 159     const gfx::Rect& visible_rect,
 160     const gfx::Size& natural_size,
 161     int32 y_stride,
 162     int32 u_stride,
 163     int32 v_stride,
 164     uint8* y_data,
 165     uint8* u_data,
 166     uint8* v_data,
 167     base::TimeDelta timestamp,
 168     const base::Closure& no_longer_needed_cb) {
 169   DCHECK(format == YV12 || format == YV16 || format == I420) << format;
 170   scoped_refptr<VideoFrame> frame(new VideoFrame(
 171       format, coded_size, visible_rect, natural_size, timestamp, false));
 172   frame->strides_[kYPlane] = y_stride;
 173   frame->strides_[kUPlane] = u_stride;
 174   frame->strides_[kVPlane] = v_stride;
 175   frame->data_[kYPlane] = y_data;
 176   frame->data_[kUPlane] = u_data;
 177   frame->data_[kVPlane] = v_data;
 178   frame->no_longer_needed_cb_ = no_longer_needed_cb;
 179   return frame;
 180 }
 181
 182 // static
 183 scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame(
 184       const scoped_refptr<VideoFrame>& frame,
 185       const base::Closure& no_longer_needed_cb) {
 186   scoped_refptr<VideoFrame> wrapped_frame(new VideoFrame(
 187       frame->format(), frame->coded_size(), frame->visible_rect(),
 188       frame->natural_size(), frame->GetTimestamp(), frame->end_of_stream()));
 189
 190   for (size_t i = 0; i < NumPlanes(frame->format()); ++i) {
 191     wrapped_frame->strides_[i] = frame->stride(i);
 192     wrapped_frame->data_[i] = frame->data(i);
 193   }
 194
 195   wrapped_frame->no_longer_needed_cb_ = no_longer_needed_cb;
 196   return wrapped_frame;
 197 }
 198
 199 // static
 200 scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() {
 201   return new VideoFrame(VideoFrame::UNKNOWN,
 202                         gfx::Size(),
 203                         gfx::Rect(),
 204                         gfx::Size(),
 205                         kNoTimestamp(),
 206                         true);
 207 }
 208
 209 // static
 210 scoped_refptr<VideoFrame> VideoFrame::CreateColorFrame(
 211     const gfx::Size& size,
 212     uint8 y, uint8 u, uint8 v,
 213     base::TimeDelta timestamp) {
 214   DCHECK(IsValidConfig(VideoFrame::YV12, size, gfx::Rect(size), size));
 215   scoped_refptr<VideoFrame> frame = VideoFrame::CreateFrame(
 216       VideoFrame::YV12, size, gfx::Rect(size), size, timestamp);
 217   FillYUV(frame.get(), y, u, v);
 218   return frame;
 219 }
 220
 221 // static
 222 scoped_refptr<VideoFrame> VideoFrame::CreateBlackFrame(const gfx::Size& size) {
 223   const uint8 kBlackY = 0x00;
 224   const uint8 kBlackUV = 0x80;
 225   const base::TimeDelta kZero;
 226   return CreateColorFrame(size, kBlackY, kBlackUV, kBlackUV, kZero);
 227 }
 228
 229 #if defined(VIDEO_HOLE)
 230 // This block and other blocks wrapped around #if defined(VIDEO_HOLE) is not
 231 // maintained by the general compositor team. Please contact the following
 232 // people instead:
 233 //
 234 // wonsik@chromium.org
 235 // ycheo@chromium.org
 236
 237 // static
 238 scoped_refptr<VideoFrame> VideoFrame::CreateHoleFrame(
 239     const gfx::Size& size) {
 240   DCHECK(IsValidConfig(VideoFrame::HOLE, size, gfx::Rect(size), size));
 241   scoped_refptr<VideoFrame> frame(new VideoFrame(
 242       VideoFrame::HOLE, size, gfx::Rect(size), size, base::TimeDelta(), false));
 243   return frame;
 244 }
 245 #endif  // defined(VIDEO_HOLE)
 246
 247 // static
 248 size_t VideoFrame::NumPlanes(Format format) {
 249   switch (format) {
 250     case VideoFrame::NATIVE_TEXTURE:
 251 #if defined(VIDEO_HOLE)
 252     case VideoFrame::HOLE:
 253 #endif  // defined(VIDEO_HOLE)
 254       return 0;
 255     case VideoFrame::YV12:
 256     case VideoFrame::YV16:
 257     case VideoFrame::I420:
 258     case VideoFrame::YV12J:
 259       return 3;
 260     case VideoFrame::YV12A:
 261       return 4;
 262     case VideoFrame::UNKNOWN:
 263     case VideoFrame::HISTOGRAM_MAX:
 264       break;
 265   }
 266   NOTREACHED() << "Unsupported video frame format: " << format;
 267   return 0;
 268 }
 269
 270 static inline size_t RoundUp(size_t value, size_t alignment) {
 271   // Check that |alignment| is a power of 2.
 272   DCHECK((alignment + (alignment - 1)) == (alignment | (alignment - 1)));
 273   return ((value + (alignment - 1)) & ~(alignment-1));
 274 }
 275
 276 // static
 277 size_t VideoFrame::AllocationSize(Format format, const gfx::Size& coded_size) {
 278   size_t total = 0;
 279   for (size_t i = 0; i < NumPlanes(format); ++i)
 280     total += PlaneAllocationSize(format, i, coded_size);
 281   return total;
 282 }
 283
 284 // static
 285 size_t VideoFrame::PlaneAllocationSize(Format format,
 286                                        size_t plane,
 287                                        const gfx::Size& coded_size) {
 288   const size_t area =
 289       RoundUp(coded_size.width(), 2) * RoundUp(coded_size.height(), 2);
 290   switch (format) {
 291     case VideoFrame::YV12:
 292     case VideoFrame::YV12J:
 293     case VideoFrame::I420: {
 294       switch (plane) {
 295         case VideoFrame::kYPlane:
 296           return area;
 297         case VideoFrame::kUPlane:
 298         case VideoFrame::kVPlane:
 299           return area / 4;
 300         default:
 301           break;
 302       }
 303     }
 304     case VideoFrame::YV12A: {
 305       switch (plane) {
 306         case VideoFrame::kYPlane:
 307         case VideoFrame::kAPlane:
 308           return area;
 309         case VideoFrame::kUPlane:
 310         case VideoFrame::kVPlane:
 311           return area / 4;
 312         default:
 313           break;
 314       }
 315     }
 316     case VideoFrame::YV16: {
 317       switch (plane) {
 318         case VideoFrame::kYPlane:
 319           return area;
 320         case VideoFrame::kUPlane:
 321         case VideoFrame::kVPlane:
 322           return area / 2;
 323         default:
 324           break;
 325       }
 326     }
 327     case VideoFrame::UNKNOWN:
 328     case VideoFrame::NATIVE_TEXTURE:
 329     case VideoFrame::HISTOGRAM_MAX:
 330 #if defined(VIDEO_HOLE)
 331     case VideoFrame::HOLE:
 332 #endif  // defined(VIDEO_HOLE)
 333       break;
 334   }
 335   NOTREACHED() << "Unsupported video frame format/plane: "
 336                << format << "/" << plane;
 337   return 0;
 338 }
 339
 340 // Release data allocated by AllocateYUV().
 341 static void ReleaseData(uint8* data) {
 342   DCHECK(data);
 343   base::AlignedFree(data);
 344 }
 345
 346 void VideoFrame::AllocateYUV() {
 347   DCHECK(format_ == VideoFrame::YV12 || format_ == VideoFrame::YV16 ||
 348          format_ == VideoFrame::YV12A || format_ == VideoFrame::I420 ||
 349          format_ == VideoFrame::YV12J);
 350   // Align Y rows at least at 16 byte boundaries.  The stride for both
 351   // YV12 and YV16 is 1/2 of the stride of Y.  For YV12, every row of bytes for
 352   // U and V applies to two rows of Y (one byte of UV for 4 bytes of Y), so in
 353   // the case of YV12 the strides are identical for the same width surface, but
 354   // the number of bytes allocated for YV12 is 1/2 the amount for U & V as
 355   // YV16. We also round the height of the surface allocated to be an even
 356   // number to avoid any potential of faulting by code that attempts to access
 357   // the Y values of the final row, but assumes that the last row of U & V
 358   // applies to a full two rows of Y. YV12A is the same as YV12, but with an
 359   // additional alpha plane that has the same size and alignment as the Y plane.
 360
 361   size_t y_stride = RoundUp(row_bytes(VideoFrame::kYPlane),
 362                             kFrameSizeAlignment);
 363   size_t uv_stride = RoundUp(row_bytes(VideoFrame::kUPlane),
 364                              kFrameSizeAlignment);
 365   // The *2 here is because some formats (e.g. h264) allow interlaced coding,
 366   // and then the size needs to be a multiple of two macroblocks (vertically).
 367   // See libavcodec/utils.c:avcodec_align_dimensions2().
 368   size_t y_height = RoundUp(coded_size_.height(), kFrameSizeAlignment * 2);
 369   size_t uv_height =
 370       (format_ == VideoFrame::YV12 || format_ == VideoFrame::YV12A ||
 371        format_ == VideoFrame::I420)
 372           ? y_height / 2
 373           : y_height;
 374   size_t y_bytes = y_height * y_stride;
 375   size_t uv_bytes = uv_height * uv_stride;
 376   size_t a_bytes = format_ == VideoFrame::YV12A ? y_bytes : 0;
 377
 378   // The extra line of UV being allocated is because h264 chroma MC
 379   // overreads by one line in some cases, see libavcodec/utils.c:
 380   // avcodec_align_dimensions2() and libavcodec/x86/h264_chromamc.asm:
 381   // put_h264_chroma_mc4_ssse3().
 382   uint8* data = reinterpret_cast<uint8*>(
 383       base::AlignedAlloc(
 384           y_bytes + (uv_bytes * 2 + uv_stride) + a_bytes + kFrameSizePadding,
 385           kFrameAddressAlignment));
 386   no_longer_needed_cb_ = base::Bind(&ReleaseData, data);
 387   COMPILE_ASSERT(0 == VideoFrame::kYPlane, y_plane_data_must_be_index_0);
 388   data_[VideoFrame::kYPlane] = data;
 389   data_[VideoFrame::kUPlane] = data + y_bytes;
 390   data_[VideoFrame::kVPlane] = data + y_bytes + uv_bytes;
 391   strides_[VideoFrame::kYPlane] = y_stride;
 392   strides_[VideoFrame::kUPlane] = uv_stride;
 393   strides_[VideoFrame::kVPlane] = uv_stride;
 394   if (format_ == YV12A) {
 395     data_[VideoFrame::kAPlane] = data + y_bytes + (2 * uv_bytes);
 396     strides_[VideoFrame::kAPlane] = y_stride;
 397   }
 398 }
 399
 400 VideoFrame::VideoFrame(VideoFrame::Format format,
 401                        const gfx::Size& coded_size,
 402                        const gfx::Rect& visible_rect,
 403                        const gfx::Size& natural_size,
 404                        base::TimeDelta timestamp,
 405                        bool end_of_stream)
 406     : format_(format),
 407       coded_size_(coded_size),
 408       visible_rect_(visible_rect),
 409       natural_size_(natural_size),
 410       shared_memory_handle_(base::SharedMemory::NULLHandle()),
 411       timestamp_(timestamp),
 412       end_of_stream_(end_of_stream) {
 413   memset(&strides_, 0, sizeof(strides_));
 414   memset(&data_, 0, sizeof(data_));
 415 }
 416
 417 VideoFrame::~VideoFrame() {
 418   if (!mailbox_holder_release_cb_.is_null()) {
 419     base::ResetAndReturn(&mailbox_holder_release_cb_)
 420         .Run(mailbox_holder_.get());
 421   }
 422   if (!no_longer_needed_cb_.is_null())
 423     base::ResetAndReturn(&no_longer_needed_cb_).Run();
 424 }
 425
 426 bool VideoFrame::IsValidPlane(size_t plane) const {
 427   return (plane < NumPlanes(format_));
 428 }
 429
 430 int VideoFrame::stride(size_t plane) const {
 431   DCHECK(IsValidPlane(plane));
 432   return strides_[plane];
 433 }
 434
 435 int VideoFrame::row_bytes(size_t plane) const {
 436   DCHECK(IsValidPlane(plane));
 437   int width = coded_size_.width();
 438   switch (format_) {
 439     // Planar, 8bpp.
 440     case YV12A:
 441       if (plane == kAPlane)
 442         return width;
 443     // Fallthrough.
 444     case YV12:
 445     case YV16:
 446     case I420:
 447     case YV12J:
 448       if (plane == kYPlane)
 449         return width;
 450       return RoundUp(width, 2) / 2;
 451
 452     default:
 453       break;
 454   }
 455
 456   // Intentionally leave out non-production formats.
 457   NOTREACHED() << "Unsupported video frame format: " << format_;
 458   return 0;
 459 }
 460
 461 int VideoFrame::rows(size_t plane) const {
 462   DCHECK(IsValidPlane(plane));
 463   int height = coded_size_.height();
 464   switch (format_) {
 465     case YV16:
 466       return height;
 467
 468     case YV12A:
 469       if (plane == kAPlane)
 470         return height;
 471     // Fallthrough.
 472     case YV12:
 473     case I420:
 474       if (plane == kYPlane)
 475         return height;
 476       return RoundUp(height, 2) / 2;
 477
 478     default:
 479       break;
 480   }
 481
 482   // Intentionally leave out non-production formats.
 483   NOTREACHED() << "Unsupported video frame format: " << format_;
 484   return 0;
 485 }
 486
 487 uint8* VideoFrame::data(size_t plane) const {
 488   DCHECK(IsValidPlane(plane));
 489   return data_[plane];
 490 }
 491
 492 gpu::MailboxHolder* VideoFrame::mailbox_holder() const {
 493   DCHECK_EQ(format_, NATIVE_TEXTURE);
 494   return mailbox_holder_.get();
 495 }
 496
 497 base::SharedMemoryHandle VideoFrame::shared_memory_handle() const {
 498   return shared_memory_handle_;
 499 }
 500
 501 void VideoFrame::HashFrameForTesting(base::MD5Context* context) {
 502   for (int plane = 0; plane < kMaxPlanes; ++plane) {
 503     if (!IsValidPlane(plane))
 504       break;
 505     for (int row = 0; row < rows(plane); ++row) {
 506       base::MD5Update(context, base::StringPiece(
 507           reinterpret_cast<char*>(data(plane) + stride(plane) * row),
 508           row_bytes(plane)));
 509     }
 510   }
 511 }
 512
 513 }  // namespace media