media/base/video_frame.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
   6 #define MEDIA_BASE_VIDEO_FRAME_H_
   7
   8 #include <vector>
   9
  10 #include "base/callback.h"
  11 #include "base/md5.h"
  12 #include "base/memory/shared_memory.h"
  13 #include "base/synchronization/lock.h"
  14 #include "gpu/command_buffer/common/mailbox_holder.h"
  15 #include "media/base/buffers.h"
  16 #include "media/base/video_frame_metadata.h"
  17 #include "media/base/video_types.h"
  18 #include "ui/gfx/geometry/rect.h"
  19 #include "ui/gfx/geometry/size.h"
  20
  21 #if defined(OS_MACOSX)
  22 #include <CoreVideo/CVPixelBuffer.h>
  23 #include "base/mac/scoped_cftyperef.h"
  24 #endif
  25
  26 namespace media {
  27
  28 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  29  public:
  30   enum {
  31     kFrameSizeAlignment = 16,
  32     kFrameSizePadding = 16,
  33     kFrameAddressAlignment = 32
  34   };
  35
  36   enum {
  37     kMaxPlanes = 4,
  38
  39     kYPlane = 0,
  40     kARGBPlane = kYPlane,
  41     kUPlane = 1,
  42     kUVPlane = kUPlane,
  43     kVPlane = 2,
  44     kAPlane = 3,
  45   };
  46
  47   // Defines the pixel storage type. Differentiates between directly accessible
  48   // |data_| and pixels that are only indirectly accessible and not via mappable
  49   // memory.
  50   // Note that VideoFrames of any StorageType can also have Texture backing,
  51   // with "classical" GPU Driver-only textures identified as STORAGE_OPAQUE.
  52   enum StorageType {
  53     STORAGE_UNKNOWN = 0,
  54     STORAGE_OPAQUE = 1,  // We don't know how VideoFrame's pixels are stored.
  55     STORAGE_UNOWNED_MEMORY = 2,  // External, non owned data pointers.
  56     STORAGE_OWNED_MEMORY = 3,  // VideoFrame has allocated its own data buffer.
  57     STORAGE_SHMEM = 4,  // Pixels are backed by Shared Memory.
  58 #if defined(OS_LINUX)
  59     // TODO(mcasas): Consider turning this type into STORAGE_NATIVE or another
  60     // meaningful name and handle it appropriately in all cases.
  61     STORAGE_DMABUFS = 5,  // Each plane is stored into a DmaBuf.
  62 #endif
  63 #if defined(VIDEO_HOLE)
  64     // Indicates protected media that needs to be directly rendered to hw. It
  65     // is, in principle, platform independent, see http://crbug.com/323157 and
  66     // https://groups.google.com/a/google.com/d/topic/chrome-gpu/eIM1RwarUmk/discussion
  67     STORAGE_HOLE = 6,
  68 #endif
  69
  70 #if defined(VIDEO_HOLE)
  71     STORAGE_LAST = STORAGE_HOLE,
  72 #elif defined(OS_LINUX)
  73     STORAGE_LAST = STORAGE_DMABUFS,
  74 #else
  75     STORAGE_LAST = STORAGE_SHMEM
  76 #endif
  77   };
  78
  79   // CB to be called on the mailbox backing this frame when the frame is
  80   // destroyed.
  81   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
  82
  83   // Interface representing client operations on a SyncPoint, i.e. insert one in
  84   // the GPU Command Buffer and wait for it.
  85   class SyncPointClient {
  86    public:
  87     SyncPointClient() {}
  88     virtual uint32 InsertSyncPoint() = 0;
  89     virtual void WaitSyncPoint(uint32 sync_point) = 0;
  90
  91    protected:
  92     virtual ~SyncPointClient() {}
  93
  94     DISALLOW_COPY_AND_ASSIGN(SyncPointClient);
  95   };
  96
  97   // Call prior to CreateFrame to ensure validity of frame configuration. Called
  98   // automatically by VideoDecoderConfig::IsValidConfig().
  99   static bool IsValidConfig(VideoPixelFormat format,
 100                             StorageType storage_type,
 101                             const gfx::Size& coded_size,
 102                             const gfx::Rect& visible_rect,
 103                             const gfx::Size& natural_size);
 104
 105   // Creates a new YUV frame in system memory with given parameters (|format|
 106   // must be YUV). Buffers for the frame are allocated but not initialized. The
 107   // caller most not make assumptions about the actual underlying size(s), but
 108   // check the returned VideoFrame instead.
 109   // TODO(mcasas): implement the RGB version of this factory method.
 110   static scoped_refptr<VideoFrame> CreateFrame(VideoPixelFormat format,
 111                                                const gfx::Size& coded_size,
 112                                                const gfx::Rect& visible_rect,
 113                                                const gfx::Size& natural_size,
 114                                                base::TimeDelta timestamp);
 115
 116   // Wraps a native texture of the given parameters with a VideoFrame.
 117   // The backing of the VideoFrame is held in the mailbox held by
 118   // |mailbox_holder|, and |mailbox_holder_release_cb| will be called with
 119   // a syncpoint as the argument when the VideoFrame is to be destroyed.
 120   static scoped_refptr<VideoFrame> WrapNativeTexture(
 121       VideoPixelFormat format,
 122       const gpu::MailboxHolder& mailbox_holder,
 123       const ReleaseMailboxCB& mailbox_holder_release_cb,
 124       const gfx::Size& coded_size,
 125       const gfx::Rect& visible_rect,
 126       const gfx::Size& natural_size,
 127       base::TimeDelta timestamp);
 128
 129   // Wraps a set of native textures representing YUV data with a VideoFrame.
 130   // |mailbox_holders_release_cb| will be called with a syncpoint as the
 131   // argument when the VideoFrame is to be destroyed.
 132   static scoped_refptr<VideoFrame> WrapYUV420NativeTextures(
 133       const gpu::MailboxHolder& y_mailbox_holder,
 134       const gpu::MailboxHolder& u_mailbox_holder,
 135       const gpu::MailboxHolder& v_mailbox_holder,
 136       const ReleaseMailboxCB& mailbox_holders_release_cb,
 137       const gfx::Size& coded_size,
 138       const gfx::Rect& visible_rect,
 139       const gfx::Size& natural_size,
 140       base::TimeDelta timestamp);
 141
 142   // Wraps packed image data residing in a memory buffer with a VideoFrame.
 143   // The image data resides in |data| and is assumed to be packed tightly in a
 144   // buffer of logical dimensions |coded_size| with the appropriate bit depth
 145   // and plane count as given by |format|. Returns NULL on failure.
 146   static scoped_refptr<VideoFrame> WrapExternalData(
 147       VideoPixelFormat format,
 148       const gfx::Size& coded_size,
 149       const gfx::Rect& visible_rect,
 150       const gfx::Size& natural_size,
 151       uint8* data,
 152       size_t data_size,
 153       base::TimeDelta timestamp);
 154
 155   // Same as WrapExternalData() with SharedMemoryHandle and its offset.
 156   static scoped_refptr<VideoFrame> WrapExternalSharedMemory(
 157       VideoPixelFormat format,
 158       const gfx::Size& coded_size,
 159       const gfx::Rect& visible_rect,
 160       const gfx::Size& natural_size,
 161       uint8* data,
 162       size_t data_size,
 163       base::SharedMemoryHandle handle,
 164       size_t shared_memory_offset,
 165       base::TimeDelta timestamp);
 166
 167   // Wraps external YUV data of the given parameters with a VideoFrame.
 168   // The returned VideoFrame does not own the data passed in.
 169   static scoped_refptr<VideoFrame> WrapExternalYuvData(
 170       VideoPixelFormat format,
 171       const gfx::Size& coded_size,
 172       const gfx::Rect& visible_rect,
 173       const gfx::Size& natural_size,
 174       int32 y_stride,
 175       int32 u_stride,
 176       int32 v_stride,
 177       uint8* y_data,
 178       uint8* u_data,
 179       uint8* v_data,
 180       base::TimeDelta timestamp);
 181
 182 #if defined(OS_LINUX)
 183   // Wraps provided dmabufs
 184   // (https://www.kernel.org/doc/Documentation/dma-buf-sharing.txt) with a
 185   // VideoFrame. The dmabuf fds are dup()ed on creation, so that the VideoFrame
 186   // retains a reference to them, and are automatically close()d on destruction,
 187   // dropping the reference. The caller may safely close() its reference after
 188   // calling WrapExternalDmabufs().
 189   // The image data is only accessible via dmabuf fds, which are usually passed
 190   // directly to a hardware device and/or to another process, or can also be
 191   // mapped via mmap() for CPU access.
 192   // Returns NULL on failure.
 193   static scoped_refptr<VideoFrame> WrapExternalDmabufs(
 194       VideoPixelFormat format,
 195       const gfx::Size& coded_size,
 196       const gfx::Rect& visible_rect,
 197       const gfx::Size& natural_size,
 198       const std::vector<int>& dmabuf_fds,
 199       base::TimeDelta timestamp);
 200 #endif
 201
 202 #if defined(OS_MACOSX)
 203   // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
 204   // retained for the lifetime of the VideoFrame and released upon destruction.
 205   // The image data is only accessible via the pixel buffer, which could be
 206   // backed by an IOSurface from another process. All the attributes of the
 207   // VideoFrame are derived from the pixel buffer, with the exception of the
 208   // timestamp. If information is missing or is incompatible (for example, a
 209   // pixel format that has no VideoFrame match), NULL is returned.
 210   // http://crbug.com/401308
 211   static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
 212       CVPixelBufferRef cv_pixel_buffer,
 213       base::TimeDelta timestamp);
 214 #endif
 215
 216   // Wraps |frame|. |visible_rect| must be a sub rect within
 217   // frame->visible_rect().
 218   static scoped_refptr<VideoFrame> WrapVideoFrame(
 219       const scoped_refptr<VideoFrame>& frame,
 220       const gfx::Rect& visible_rect,
 221       const gfx::Size& natural_size);
 222
 223   // Creates a frame which indicates end-of-stream.
 224   static scoped_refptr<VideoFrame> CreateEOSFrame();
 225
 226   // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
 227   static scoped_refptr<VideoFrame> CreateColorFrame(
 228       const gfx::Size& size,
 229       uint8 y, uint8 u, uint8 v,
 230       base::TimeDelta timestamp);
 231
 232   // Allocates YV12 frame based on |size|, and sets its data to the YUV
 233   // equivalent of RGB(0,0,0).
 234   static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
 235
 236   // Allocates YV12A frame based on |size|, and sets its data to the YUVA
 237   // equivalent of RGBA(0,0,0,0).
 238   static scoped_refptr<VideoFrame> CreateTransparentFrame(
 239       const gfx::Size& size);
 240
 241 #if defined(VIDEO_HOLE)
 242   // Allocates a hole frame.
 243   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 244 #endif  // defined(VIDEO_HOLE)
 245
 246   static size_t NumPlanes(VideoPixelFormat format);
 247
 248   // Returns the required allocation size for a (tightly packed) frame of the
 249   // given coded size and format.
 250   static size_t AllocationSize(VideoPixelFormat format,
 251                                const gfx::Size& coded_size);
 252
 253   // Returns the plane gfx::Size (in bytes) for a plane of the given coded size
 254   // and format.
 255   static gfx::Size PlaneSize(VideoPixelFormat format,
 256                              size_t plane,
 257                              const gfx::Size& coded_size);
 258
 259   // Returns horizontal bits per pixel for given |plane| and |format|.
 260   static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane);
 261
 262   // Returns bits per pixel for given |plane| and |format|.
 263   static int PlaneBitsPerPixel(VideoPixelFormat format, size_t plane);
 264
 265   // Returns the number of bytes per row for the given plane, format, and width.
 266   // The width may be aligned to format requirements.
 267   static size_t RowBytes(size_t plane, VideoPixelFormat format, int width);
 268
 269   // Returns the number of rows for the given plane, format, and height.
 270   // The height may be aligned to format requirements.
 271   static size_t Rows(size_t plane, VideoPixelFormat format, int height);
 272
 273   // Returns the number of columns for the given plane, format, and width.
 274   // The width may be aligned to format requirements.
 275   static size_t Columns(size_t plane, VideoPixelFormat format, int width);
 276
 277   // Used to keep a running hash of seen frames.  Expects an initialized MD5
 278   // context.  Calls MD5Update with the context and the contents of the frame.
 279   static void HashFrameForTesting(base::MD5Context* context,
 280                                   const scoped_refptr<VideoFrame>& frame);
 281
 282   // Returns true if |frame| is accessible and mapped in the VideoFrame memory
 283   // space. If false, clients should refrain from accessing data(),
 284   // visible_data() etc.
 285   bool IsMappable() const;
 286
 287   // Returns true if |frame| has textures with any StorageType and should not be
 288   // accessed via data(), visible_data() etc.
 289   bool HasTextures() const;
 290
 291   VideoPixelFormat format() const { return format_; }
 292   StorageType storage_type() const { return storage_type_; }
 293
 294   const gfx::Size& coded_size() const { return coded_size_; }
 295   const gfx::Rect& visible_rect() const { return visible_rect_; }
 296   const gfx::Size& natural_size() const { return natural_size_; }
 297
 298   int stride(size_t plane) const;
 299
 300   // Returns the number of bytes per row and number of rows for a given plane.
 301   //
 302   // As opposed to stride(), row_bytes() refers to the bytes representing
 303   // frame data scanlines (coded_size.width() pixels, without stride padding).
 304   int row_bytes(size_t plane) const;
 305   int rows(size_t plane) const;
 306
 307   // Returns pointer to the buffer for a given plane, if this is an
 308   // IsMappable() frame type. The memory is owned by VideoFrame object and must
 309   // not be freed by the caller.
 310   const uint8* data(size_t plane) const;
 311   uint8* data(size_t plane);
 312
 313   // Returns pointer to the data in the visible region of the frame, for
 314   // IsMappable() storage types. The returned pointer is offsetted into the
 315   // plane buffer specified by visible_rect().origin(). Memory is owned by
 316   // VideoFrame object and must not be freed by the caller.
 317   const uint8* visible_data(size_t plane) const;
 318   uint8* visible_data(size_t plane);
 319
 320   // Returns a mailbox holder for a given texture.
 321   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
 322   // mailbox, the caller must wait for the included sync point.
 323   const gpu::MailboxHolder& mailbox_holder(size_t texture_index) const;
 324
 325   // Returns the shared-memory handle, if present
 326   base::SharedMemoryHandle shared_memory_handle() const;
 327
 328   // Returns the offset into the shared memory where the frame data begins.
 329   size_t shared_memory_offset() const;
 330
 331 #if defined(OS_LINUX)
 332   // Returns backing DmaBuf file descriptor for given |plane|, if present, or
 333   // -1 if not.
 334   // TODO(mcasas): Rename to DmabufFd() to comply with Style Guide.
 335   int dmabuf_fd(size_t plane) const;
 336
 337   // Duplicates internally the |fds_in|, overwriting the current ones. Returns
 338   // false if something goes wrong, and leaves all internal fds closed.
 339   bool DuplicateFileDescriptors(const std::vector<int>& fds_in);
 340 #endif
 341
 342   void AddSharedMemoryHandle(base::SharedMemoryHandle handle);
 343
 344 #if defined(OS_MACOSX)
 345   // Returns the backing CVPixelBuffer, if present.
 346   // TODO(mcasas): Rename to CvPixelBuffer() to comply with Style Guide.
 347   CVPixelBufferRef cv_pixel_buffer() const;
 348 #endif
 349
 350   // Adds a callback to be run when the VideoFrame is about to be destroyed.
 351   // The callback may be run from ANY THREAD, and so it is up to the client to
 352   // ensure thread safety.  Although read-only access to the members of this
 353   // VideoFrame is permitted while the callback executes (including
 354   // VideoFrameMetadata), clients should not assume the data pointers are
 355   // valid.
 356   void AddDestructionObserver(const base::Closure& callback);
 357
 358   // Returns a dictionary of optional metadata.  This contains information
 359   // associated with the frame that downstream clients might use for frame-level
 360   // logging, quality/performance optimizations, signaling, etc.
 361   //
 362   // TODO(miu): Move some of the "extra" members of VideoFrame (below) into
 363   // here as a later clean-up step.
 364   const VideoFrameMetadata* metadata() const { return &metadata_; }
 365   VideoFrameMetadata* metadata() { return &metadata_; }
 366
 367   base::TimeDelta timestamp() const { return timestamp_; }
 368   void set_timestamp(base::TimeDelta timestamp) {
 369     timestamp_ = timestamp;
 370   }
 371
 372   // It uses |client| to insert a new sync point and potentially waits on a
 373   // older sync point. The final sync point will be used to release this
 374   // VideoFrame.
 375   // This method is thread safe. Both blink and compositor threads can call it.
 376   void UpdateReleaseSyncPoint(SyncPointClient* client);
 377
 378  private:
 379   friend class base::RefCountedThreadSafe<VideoFrame>;
 380
 381   static scoped_refptr<VideoFrame> WrapExternalStorage(
 382       VideoPixelFormat format,
 383       StorageType storage_type,
 384       const gfx::Size& coded_size,
 385       const gfx::Rect& visible_rect,
 386       const gfx::Size& natural_size,
 387       uint8* data,
 388       size_t data_size,
 389       base::TimeDelta timestamp,
 390       base::SharedMemoryHandle handle,
 391       size_t data_offset);
 392
 393   // Clients must use the static factory/wrapping methods to create a new frame.
 394   VideoFrame(VideoPixelFormat format,
 395              StorageType storage_type,
 396              const gfx::Size& coded_size,
 397              const gfx::Rect& visible_rect,
 398              const gfx::Size& natural_size,
 399              base::TimeDelta timestamp);
 400   VideoFrame(VideoPixelFormat format,
 401              StorageType storage_type,
 402              const gfx::Size& coded_size,
 403              const gfx::Rect& visible_rect,
 404              const gfx::Size& natural_size,
 405              base::TimeDelta timestamp,
 406              base::SharedMemoryHandle handle,
 407              size_t shared_memory_offset);
 408   VideoFrame(VideoPixelFormat format,
 409              StorageType storage_type,
 410              const gfx::Size& coded_size,
 411              const gfx::Rect& visible_rect,
 412              const gfx::Size& natural_size,
 413              const gpu::MailboxHolder(&mailbox_holders)[kMaxPlanes],
 414              const ReleaseMailboxCB& mailbox_holder_release_cb,
 415              base::TimeDelta timestamp);
 416   virtual ~VideoFrame();
 417
 418   void AllocateYUV();
 419
 420   // Frame format.
 421   const VideoPixelFormat format_;
 422
 423   // Storage type for the different planes.
 424   StorageType storage_type_;  // TODO(mcasas): make const
 425
 426   // Width and height of the video frame, in pixels. This must include pixel
 427   // data for the whole image; i.e. for YUV formats with subsampled chroma
 428   // planes, in the case that the visible portion of the image does not line up
 429   // on a sample boundary, |coded_size_| must be rounded up appropriately and
 430   // the pixel data provided for the odd pixels.
 431   const gfx::Size coded_size_;
 432
 433   // Width, height, and offsets of the visible portion of the video frame. Must
 434   // be a subrect of |coded_size_|. Can be odd with respect to the sample
 435   // boundaries, e.g. for formats with subsampled chroma.
 436   const gfx::Rect visible_rect_;
 437
 438   // Width and height of the visible portion of the video frame
 439   // (|visible_rect_.size()|) with aspect ratio taken into account.
 440   const gfx::Size natural_size_;
 441
 442   // Array of strides for each plane, typically greater or equal to the width
 443   // of the surface divided by the horizontal sampling period.  Note that
 444   // strides can be negative.
 445   int32 strides_[kMaxPlanes];
 446
 447   // Array of data pointers to each plane.
 448   // TODO(mcasas): we don't know on ctor if we own |data_| or not. After
 449   // refactoring VideoFrame, change to scoped_ptr<uint8, AlignedFreeDeleter>.
 450   uint8* data_[kMaxPlanes];
 451
 452   // Native texture mailboxes, if this is a IsTexture() frame.
 453   gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
 454   ReleaseMailboxCB mailbox_holders_release_cb_;
 455
 456   // Shared memory handle and associated offset inside it, if this frame is
 457   // a STORAGE_SHMEM one.
 458   base::SharedMemoryHandle shared_memory_handle_;
 459   size_t shared_memory_offset_;
 460
 461 #if defined(OS_LINUX)
 462   // Dmabufs for each plane. If set, this frame has DmaBuf backing in some way.
 463   base::ScopedFD dmabuf_fds_[kMaxPlanes];
 464 #endif
 465
 466 #if defined(OS_MACOSX)
 467   // CVPixelBuffer, if this frame is wrapping one.
 468   base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
 469 #endif
 470
 471   std::vector<base::Closure> done_callbacks_;
 472
 473   base::TimeDelta timestamp_;
 474
 475   base::Lock release_sync_point_lock_;
 476   uint32 release_sync_point_;
 477
 478   VideoFrameMetadata metadata_;
 479
 480   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 481 };
 482
 483 }  // namespace media
 484
 485 #endif  // MEDIA_BASE_VIDEO_FRAME_H_