media/base/video_frame.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
   6 #define MEDIA_BASE_VIDEO_FRAME_H_
   7
   8 #include <vector>
   9
  10 #include "base/callback.h"
  11 #include "base/md5.h"
  12 #include "base/memory/shared_memory.h"
  13 #include "base/synchronization/lock.h"
  14 #include "gpu/command_buffer/common/mailbox_holder.h"
  15 #include "media/base/video_frame_metadata.h"
  16 #include "media/base/video_types.h"
  17 #include "ui/gfx/geometry/rect.h"
  18 #include "ui/gfx/geometry/size.h"
  19
  20 #if defined(OS_MACOSX)
  21 #include <CoreVideo/CVPixelBuffer.h>
  22 #include "base/mac/scoped_cftyperef.h"
  23 #endif
  24
  25 namespace media {
  26
  27 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  28  public:
  29   enum {
  30     kFrameSizeAlignment = 16,
  31     kFrameSizePadding = 16,
  32     kFrameAddressAlignment = 32
  33   };
  34
  35   enum {
  36     kMaxPlanes = 4,
  37
  38     kYPlane = 0,
  39     kARGBPlane = kYPlane,
  40     kUPlane = 1,
  41     kUVPlane = kUPlane,
  42     kVPlane = 2,
  43     kAPlane = 3,
  44   };
  45
  46   // Defines the pixel storage type. Differentiates between directly accessible
  47   // |data_| and pixels that are only indirectly accessible and not via mappable
  48   // memory.
  49   // Note that VideoFrames of any StorageType can also have Texture backing,
  50   // with "classical" GPU Driver-only textures identified as STORAGE_OPAQUE.
  51   enum StorageType {
  52     STORAGE_UNKNOWN = 0,
  53     STORAGE_OPAQUE = 1,  // We don't know how VideoFrame's pixels are stored.
  54     STORAGE_UNOWNED_MEMORY = 2,  // External, non owned data pointers.
  55     STORAGE_OWNED_MEMORY = 3,  // VideoFrame has allocated its own data buffer.
  56     STORAGE_SHMEM = 4,  // Pixels are backed by Shared Memory.
  57 #if defined(OS_LINUX)
  58     // TODO(mcasas): Consider turning this type into STORAGE_NATIVE or another
  59     // meaningful name and handle it appropriately in all cases.
  60     STORAGE_DMABUFS = 5,  // Each plane is stored into a DmaBuf.
  61 #endif
  62 #if defined(VIDEO_HOLE)
  63     // Indicates protected media that needs to be directly rendered to hw. It
  64     // is, in principle, platform independent, see http://crbug.com/323157 and
  65     // https://groups.google.com/a/google.com/d/topic/chrome-gpu/eIM1RwarUmk/discussion
  66     STORAGE_HOLE = 6,
  67 #endif
  68
  69 #if defined(VIDEO_HOLE)
  70     STORAGE_LAST = STORAGE_HOLE,
  71 #elif defined(OS_LINUX)
  72     STORAGE_LAST = STORAGE_DMABUFS,
  73 #else
  74     STORAGE_LAST = STORAGE_SHMEM
  75 #endif
  76   };
  77
  78   // CB to be called on the mailbox backing this frame when the frame is
  79   // destroyed.
  80   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
  81
  82   // Interface representing client operations on a SyncPoint, i.e. insert one in
  83   // the GPU Command Buffer and wait for it.
  84   class SyncPointClient {
  85    public:
  86     SyncPointClient() {}
  87     virtual uint32 InsertSyncPoint() = 0;
  88     virtual void WaitSyncPoint(uint32 sync_point) = 0;
  89
  90    protected:
  91     virtual ~SyncPointClient() {}
  92
  93     DISALLOW_COPY_AND_ASSIGN(SyncPointClient);
  94   };
  95
  96   // Call prior to CreateFrame to ensure validity of frame configuration. Called
  97   // automatically by VideoDecoderConfig::IsValidConfig().
  98   static bool IsValidConfig(VideoPixelFormat format,
  99                             StorageType storage_type,
 100                             const gfx::Size& coded_size,
 101                             const gfx::Rect& visible_rect,
 102                             const gfx::Size& natural_size);
 103
 104   // Creates a new YUV frame in system memory with given parameters (|format|
 105   // must be YUV). Buffers for the frame are allocated but not initialized. The
 106   // caller most not make assumptions about the actual underlying size(s), but
 107   // check the returned VideoFrame instead.
 108   // TODO(mcasas): implement the RGB version of this factory method.
 109   static scoped_refptr<VideoFrame> CreateFrame(VideoPixelFormat format,
 110                                                const gfx::Size& coded_size,
 111                                                const gfx::Rect& visible_rect,
 112                                                const gfx::Size& natural_size,
 113                                                base::TimeDelta timestamp);
 114
 115   // Offers the same functionality as CreateFrame, and additionally zeroes out
 116   // the initial allocated buffers.
 117   static scoped_refptr<VideoFrame> CreateZeroInitializedFrame(
 118       VideoPixelFormat format,
 119       const gfx::Size& coded_size,
 120       const gfx::Rect& visible_rect,
 121       const gfx::Size& natural_size,
 122       base::TimeDelta timestamp);
 123
 124   // Wraps a native texture of the given parameters with a VideoFrame.
 125   // The backing of the VideoFrame is held in the mailbox held by
 126   // |mailbox_holder|, and |mailbox_holder_release_cb| will be called with
 127   // a syncpoint as the argument when the VideoFrame is to be destroyed.
 128   static scoped_refptr<VideoFrame> WrapNativeTexture(
 129       VideoPixelFormat format,
 130       const gpu::MailboxHolder& mailbox_holder,
 131       const ReleaseMailboxCB& mailbox_holder_release_cb,
 132       const gfx::Size& coded_size,
 133       const gfx::Rect& visible_rect,
 134       const gfx::Size& natural_size,
 135       base::TimeDelta timestamp);
 136
 137   // Wraps a set of native textures representing YUV data with a VideoFrame.
 138   // |mailbox_holders_release_cb| will be called with a syncpoint as the
 139   // argument when the VideoFrame is to be destroyed.
 140   static scoped_refptr<VideoFrame> WrapYUV420NativeTextures(
 141       const gpu::MailboxHolder& y_mailbox_holder,
 142       const gpu::MailboxHolder& u_mailbox_holder,
 143       const gpu::MailboxHolder& v_mailbox_holder,
 144       const ReleaseMailboxCB& mailbox_holders_release_cb,
 145       const gfx::Size& coded_size,
 146       const gfx::Rect& visible_rect,
 147       const gfx::Size& natural_size,
 148       base::TimeDelta timestamp);
 149
 150   // Wraps packed image data residing in a memory buffer with a VideoFrame.
 151   // The image data resides in |data| and is assumed to be packed tightly in a
 152   // buffer of logical dimensions |coded_size| with the appropriate bit depth
 153   // and plane count as given by |format|. Returns NULL on failure.
 154   static scoped_refptr<VideoFrame> WrapExternalData(
 155       VideoPixelFormat format,
 156       const gfx::Size& coded_size,
 157       const gfx::Rect& visible_rect,
 158       const gfx::Size& natural_size,
 159       uint8* data,
 160       size_t data_size,
 161       base::TimeDelta timestamp);
 162
 163   // Same as WrapExternalData() with SharedMemoryHandle and its offset.
 164   static scoped_refptr<VideoFrame> WrapExternalSharedMemory(
 165       VideoPixelFormat format,
 166       const gfx::Size& coded_size,
 167       const gfx::Rect& visible_rect,
 168       const gfx::Size& natural_size,
 169       uint8* data,
 170       size_t data_size,
 171       base::SharedMemoryHandle handle,
 172       size_t shared_memory_offset,
 173       base::TimeDelta timestamp);
 174
 175   // Wraps external YUV data of the given parameters with a VideoFrame.
 176   // The returned VideoFrame does not own the data passed in.
 177   static scoped_refptr<VideoFrame> WrapExternalYuvData(
 178       VideoPixelFormat format,
 179       const gfx::Size& coded_size,
 180       const gfx::Rect& visible_rect,
 181       const gfx::Size& natural_size,
 182       int32 y_stride,
 183       int32 u_stride,
 184       int32 v_stride,
 185       uint8* y_data,
 186       uint8* u_data,
 187       uint8* v_data,
 188       base::TimeDelta timestamp);
 189
 190 #if defined(OS_LINUX)
 191   // Wraps provided dmabufs
 192   // (https://www.kernel.org/doc/Documentation/dma-buf-sharing.txt) with a
 193   // VideoFrame. The dmabuf fds are dup()ed on creation, so that the VideoFrame
 194   // retains a reference to them, and are automatically close()d on destruction,
 195   // dropping the reference. The caller may safely close() its reference after
 196   // calling WrapExternalDmabufs().
 197   // The image data is only accessible via dmabuf fds, which are usually passed
 198   // directly to a hardware device and/or to another process, or can also be
 199   // mapped via mmap() for CPU access.
 200   // Returns NULL on failure.
 201   static scoped_refptr<VideoFrame> WrapExternalDmabufs(
 202       VideoPixelFormat format,
 203       const gfx::Size& coded_size,
 204       const gfx::Rect& visible_rect,
 205       const gfx::Size& natural_size,
 206       const std::vector<int>& dmabuf_fds,
 207       base::TimeDelta timestamp);
 208 #endif
 209
 210 #if defined(OS_MACOSX)
 211   // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
 212   // retained for the lifetime of the VideoFrame and released upon destruction.
 213   // The image data is only accessible via the pixel buffer, which could be
 214   // backed by an IOSurface from another process. All the attributes of the
 215   // VideoFrame are derived from the pixel buffer, with the exception of the
 216   // timestamp. If information is missing or is incompatible (for example, a
 217   // pixel format that has no VideoFrame match), NULL is returned.
 218   // http://crbug.com/401308
 219   static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
 220       CVPixelBufferRef cv_pixel_buffer,
 221       base::TimeDelta timestamp);
 222 #endif
 223
 224   // Wraps |frame|. |visible_rect| must be a sub rect within
 225   // frame->visible_rect().
 226   static scoped_refptr<VideoFrame> WrapVideoFrame(
 227       const scoped_refptr<VideoFrame>& frame,
 228       const gfx::Rect& visible_rect,
 229       const gfx::Size& natural_size);
 230
 231   // Creates a frame which indicates end-of-stream.
 232   static scoped_refptr<VideoFrame> CreateEOSFrame();
 233
 234   // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
 235   static scoped_refptr<VideoFrame> CreateColorFrame(
 236       const gfx::Size& size,
 237       uint8 y, uint8 u, uint8 v,
 238       base::TimeDelta timestamp);
 239
 240   // Allocates YV12 frame based on |size|, and sets its data to the YUV
 241   // equivalent of RGB(0,0,0).
 242   static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
 243
 244   // Allocates YV12A frame based on |size|, and sets its data to the YUVA
 245   // equivalent of RGBA(0,0,0,0).
 246   static scoped_refptr<VideoFrame> CreateTransparentFrame(
 247       const gfx::Size& size);
 248
 249 #if defined(VIDEO_HOLE)
 250   // Allocates a hole frame.
 251   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 252 #endif  // defined(VIDEO_HOLE)
 253
 254   static size_t NumPlanes(VideoPixelFormat format);
 255
 256   // Returns the required allocation size for a (tightly packed) frame of the
 257   // given coded size and format.
 258   static size_t AllocationSize(VideoPixelFormat format,
 259                                const gfx::Size& coded_size);
 260
 261   // Returns the plane gfx::Size (in bytes) for a plane of the given coded size
 262   // and format.
 263   static gfx::Size PlaneSize(VideoPixelFormat format,
 264                              size_t plane,
 265                              const gfx::Size& coded_size);
 266
 267   // Returns horizontal bits per pixel for given |plane| and |format|.
 268   static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane);
 269
 270   // Returns bits per pixel for given |plane| and |format|.
 271   static int PlaneBitsPerPixel(VideoPixelFormat format, size_t plane);
 272
 273   // Returns the number of bytes per row for the given plane, format, and width.
 274   // The width may be aligned to format requirements.
 275   static size_t RowBytes(size_t plane, VideoPixelFormat format, int width);
 276
 277   // Returns the number of rows for the given plane, format, and height.
 278   // The height may be aligned to format requirements.
 279   static size_t Rows(size_t plane, VideoPixelFormat format, int height);
 280
 281   // Returns the number of columns for the given plane, format, and width.
 282   // The width may be aligned to format requirements.
 283   static size_t Columns(size_t plane, VideoPixelFormat format, int width);
 284
 285   // Used to keep a running hash of seen frames.  Expects an initialized MD5
 286   // context.  Calls MD5Update with the context and the contents of the frame.
 287   static void HashFrameForTesting(base::MD5Context* context,
 288                                   const scoped_refptr<VideoFrame>& frame);
 289
 290   // Returns true if |frame| is accessible and mapped in the VideoFrame memory
 291   // space. If false, clients should refrain from accessing data(),
 292   // visible_data() etc.
 293   bool IsMappable() const;
 294
 295   // Returns true if |frame| has textures with any StorageType and should not be
 296   // accessed via data(), visible_data() etc.
 297   bool HasTextures() const;
 298
 299   VideoPixelFormat format() const { return format_; }
 300   StorageType storage_type() const { return storage_type_; }
 301
 302   const gfx::Size& coded_size() const { return coded_size_; }
 303   const gfx::Rect& visible_rect() const { return visible_rect_; }
 304   const gfx::Size& natural_size() const { return natural_size_; }
 305
 306   int stride(size_t plane) const;
 307
 308   // Returns the number of bytes per row and number of rows for a given plane.
 309   //
 310   // As opposed to stride(), row_bytes() refers to the bytes representing
 311   // frame data scanlines (coded_size.width() pixels, without stride padding).
 312   int row_bytes(size_t plane) const;
 313   int rows(size_t plane) const;
 314
 315   // Returns pointer to the buffer for a given plane, if this is an
 316   // IsMappable() frame type. The memory is owned by VideoFrame object and must
 317   // not be freed by the caller.
 318   const uint8* data(size_t plane) const;
 319   uint8* data(size_t plane);
 320
 321   // Returns pointer to the data in the visible region of the frame, for
 322   // IsMappable() storage types. The returned pointer is offsetted into the
 323   // plane buffer specified by visible_rect().origin(). Memory is owned by
 324   // VideoFrame object and must not be freed by the caller.
 325   const uint8* visible_data(size_t plane) const;
 326   uint8* visible_data(size_t plane);
 327
 328   // Returns a mailbox holder for a given texture.
 329   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
 330   // mailbox, the caller must wait for the included sync point.
 331   const gpu::MailboxHolder& mailbox_holder(size_t texture_index) const;
 332
 333   // Returns the shared-memory handle, if present
 334   base::SharedMemoryHandle shared_memory_handle() const;
 335
 336   // Returns the offset into the shared memory where the frame data begins.
 337   size_t shared_memory_offset() const;
 338
 339 #if defined(OS_LINUX)
 340   // Returns backing DmaBuf file descriptor for given |plane|, if present, or
 341   // -1 if not.
 342   // TODO(mcasas): Rename to DmabufFd() to comply with Style Guide.
 343   int dmabuf_fd(size_t plane) const;
 344
 345   // Duplicates internally the |fds_in|, overwriting the current ones. Returns
 346   // false if something goes wrong, and leaves all internal fds closed.
 347   bool DuplicateFileDescriptors(const std::vector<int>& fds_in);
 348 #endif
 349
 350   void AddSharedMemoryHandle(base::SharedMemoryHandle handle);
 351
 352 #if defined(OS_MACOSX)
 353   // Returns the backing CVPixelBuffer, if present.
 354   // TODO(mcasas): Rename to CvPixelBuffer() to comply with Style Guide.
 355   CVPixelBufferRef cv_pixel_buffer() const;
 356 #endif
 357
 358   // Adds a callback to be run when the VideoFrame is about to be destroyed.
 359   // The callback may be run from ANY THREAD, and so it is up to the client to
 360   // ensure thread safety.  Although read-only access to the members of this
 361   // VideoFrame is permitted while the callback executes (including
 362   // VideoFrameMetadata), clients should not assume the data pointers are
 363   // valid.
 364   void AddDestructionObserver(const base::Closure& callback);
 365
 366   // Returns a dictionary of optional metadata.  This contains information
 367   // associated with the frame that downstream clients might use for frame-level
 368   // logging, quality/performance optimizations, signaling, etc.
 369   //
 370   // TODO(miu): Move some of the "extra" members of VideoFrame (below) into
 371   // here as a later clean-up step.
 372   const VideoFrameMetadata* metadata() const { return &metadata_; }
 373   VideoFrameMetadata* metadata() { return &metadata_; }
 374
 375   // The time span between the current frame and the first frame of the stream.
 376   // This is the media timestamp, and not the reference time.
 377   // See VideoFrameMetadata::REFERENCE_TIME for details.
 378   base::TimeDelta timestamp() const { return timestamp_; }
 379   void set_timestamp(base::TimeDelta timestamp) {
 380     timestamp_ = timestamp;
 381   }
 382
 383   // It uses |client| to insert a new sync point and potentially waits on a
 384   // older sync point. The final sync point will be used to release this
 385   // VideoFrame.
 386   // This method is thread safe. Both blink and compositor threads can call it.
 387   void UpdateReleaseSyncPoint(SyncPointClient* client);
 388
 389  private:
 390   friend class base::RefCountedThreadSafe<VideoFrame>;
 391
 392   static scoped_refptr<VideoFrame> WrapExternalStorage(
 393       VideoPixelFormat format,
 394       StorageType storage_type,
 395       const gfx::Size& coded_size,
 396       const gfx::Rect& visible_rect,
 397       const gfx::Size& natural_size,
 398       uint8* data,
 399       size_t data_size,
 400       base::TimeDelta timestamp,
 401       base::SharedMemoryHandle handle,
 402       size_t data_offset);
 403
 404   // Clients must use the static factory/wrapping methods to create a new frame.
 405   VideoFrame(VideoPixelFormat format,
 406              StorageType storage_type,
 407              const gfx::Size& coded_size,
 408              const gfx::Rect& visible_rect,
 409              const gfx::Size& natural_size,
 410              base::TimeDelta timestamp);
 411   VideoFrame(VideoPixelFormat format,
 412              StorageType storage_type,
 413              const gfx::Size& coded_size,
 414              const gfx::Rect& visible_rect,
 415              const gfx::Size& natural_size,
 416              base::TimeDelta timestamp,
 417              base::SharedMemoryHandle handle,
 418              size_t shared_memory_offset);
 419   VideoFrame(VideoPixelFormat format,
 420              StorageType storage_type,
 421              const gfx::Size& coded_size,
 422              const gfx::Rect& visible_rect,
 423              const gfx::Size& natural_size,
 424              const gpu::MailboxHolder(&mailbox_holders)[kMaxPlanes],
 425              const ReleaseMailboxCB& mailbox_holder_release_cb,
 426              base::TimeDelta timestamp);
 427   virtual ~VideoFrame();
 428
 429   static scoped_refptr<VideoFrame> CreateFrameInternal(
 430       VideoPixelFormat format,
 431       const gfx::Size& coded_size,
 432       const gfx::Rect& visible_rect,
 433       const gfx::Size& natural_size,
 434       base::TimeDelta timestamp,
 435       bool zero_initialize_memory);
 436
 437   void AllocateYUV(bool zero_initialize_memory);
 438
 439   // Frame format.
 440   const VideoPixelFormat format_;
 441
 442   // Storage type for the different planes.
 443   StorageType storage_type_;  // TODO(mcasas): make const
 444
 445   // Width and height of the video frame, in pixels. This must include pixel
 446   // data for the whole image; i.e. for YUV formats with subsampled chroma
 447   // planes, in the case that the visible portion of the image does not line up
 448   // on a sample boundary, |coded_size_| must be rounded up appropriately and
 449   // the pixel data provided for the odd pixels.
 450   const gfx::Size coded_size_;
 451
 452   // Width, height, and offsets of the visible portion of the video frame. Must
 453   // be a subrect of |coded_size_|. Can be odd with respect to the sample
 454   // boundaries, e.g. for formats with subsampled chroma.
 455   const gfx::Rect visible_rect_;
 456
 457   // Width and height of the visible portion of the video frame
 458   // (|visible_rect_.size()|) with aspect ratio taken into account.
 459   const gfx::Size natural_size_;
 460
 461   // Array of strides for each plane, typically greater or equal to the width
 462   // of the surface divided by the horizontal sampling period.  Note that
 463   // strides can be negative.
 464   int32 strides_[kMaxPlanes];
 465
 466   // Array of data pointers to each plane.
 467   // TODO(mcasas): we don't know on ctor if we own |data_| or not. After
 468   // refactoring VideoFrame, change to scoped_ptr<uint8, AlignedFreeDeleter>.
 469   uint8* data_[kMaxPlanes];
 470
 471   // Native texture mailboxes, if this is a IsTexture() frame.
 472   gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
 473   ReleaseMailboxCB mailbox_holders_release_cb_;
 474
 475   // Shared memory handle and associated offset inside it, if this frame is
 476   // a STORAGE_SHMEM one.
 477   base::SharedMemoryHandle shared_memory_handle_;
 478   size_t shared_memory_offset_;
 479
 480 #if defined(OS_LINUX)
 481   // Dmabufs for each plane. If set, this frame has DmaBuf backing in some way.
 482   base::ScopedFD dmabuf_fds_[kMaxPlanes];
 483 #endif
 484
 485 #if defined(OS_MACOSX)
 486   // CVPixelBuffer, if this frame is wrapping one.
 487   base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
 488 #endif
 489
 490   std::vector<base::Closure> done_callbacks_;
 491
 492   base::TimeDelta timestamp_;
 493
 494   base::Lock release_sync_point_lock_;
 495   uint32 release_sync_point_;
 496
 497   VideoFrameMetadata metadata_;
 498
 499   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 500 };
 501
 502 }  // namespace media
 503
 504 #endif  // MEDIA_BASE_VIDEO_FRAME_H_