media/base/video_frame.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
   6 #define MEDIA_BASE_VIDEO_FRAME_H_
   7
   8 #include <vector>
   9
  10 #include "base/callback.h"
  11 #include "base/md5.h"
  12 #include "base/memory/shared_memory.h"
  13 #include "base/synchronization/lock.h"
  14 #include "gpu/command_buffer/common/mailbox_holder.h"
  15 #include "media/base/buffers.h"
  16 #include "media/base/video_frame_metadata.h"
  17 #include "media/base/video_types.h"
  18 #include "ui/gfx/geometry/rect.h"
  19 #include "ui/gfx/geometry/size.h"
  20
  21 #if defined(OS_MACOSX)
  22 #include <CoreVideo/CVPixelBuffer.h>
  23 #include "base/mac/scoped_cftyperef.h"
  24 #endif
  25
  26 namespace media {
  27
  28 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  29  public:
  30   enum {
  31     kFrameSizeAlignment = 16,
  32     kFrameSizePadding = 16,
  33     kFrameAddressAlignment = 32
  34   };
  35
  36   enum {
  37     kMaxPlanes = 4,
  38
  39     kYPlane = 0,
  40     kARGBPlane = kYPlane,
  41     kUPlane = 1,
  42     kUVPlane = kUPlane,
  43     kVPlane = 2,
  44     kAPlane = 3,
  45   };
  46
  47   // Defines the pixel storage type. Differentiates between directly accessible
  48   // |data_| and pixels that are only indirectly accessible and not via mappable
  49   // memory.
  50   // Note that VideoFrames of any StorageType can also have Texture backing,
  51   // with "classical" GPU Driver-only textures identified as STORAGE_OPAQUE.
  52   enum StorageType {
  53     STORAGE_UNKNOWN = 0,
  54     STORAGE_OPAQUE = 1,  // We don't know how VideoFrame's pixels are stored.
  55     STORAGE_UNOWNED_MEMORY = 2,  // External, non owned data pointers.
  56     STORAGE_OWNED_MEMORY = 3,  // VideoFrame has allocated its own data buffer.
  57     STORAGE_SHMEM = 4,  // Pixels are backed by Shared Memory.
  58 #if defined(OS_LINUX)
  59     // TODO(mcasas): Consider turning this type into STORAGE_NATIVE or another
  60     // meaningful name and handle it appropriately in all cases.
  61     STORAGE_DMABUFS = 5,  // Each plane is stored into a DmaBuf.
  62 #endif
  63 #if defined(VIDEO_HOLE)
  64     // Indicates protected media that needs to be directly rendered to hw. It
  65     // is, in principle, platform independent, see http://crbug.com/323157 and
  66     // https://groups.google.com/a/google.com/d/topic/chrome-gpu/eIM1RwarUmk/discussion
  67     STORAGE_HOLE = 6,
  68 #endif
  69
  70 #if defined(VIDEO_HOLE)
  71     STORAGE_LAST = STORAGE_HOLE,
  72 #elif defined(OS_LINUX)
  73     STORAGE_LAST = STORAGE_DMABUFS,
  74 #else
  75     STORAGE_LAST = STORAGE_SHMEM
  76 #endif
  77   };
  78
  79   // CB to be called on the mailbox backing this frame when the frame is
  80   // destroyed.
  81   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
  82
  83   // Interface representing client operations on a SyncPoint, i.e. insert one in
  84   // the GPU Command Buffer and wait for it.
  85   class SyncPointClient {
  86    public:
  87     SyncPointClient() {}
  88     virtual uint32 InsertSyncPoint() = 0;
  89     virtual void WaitSyncPoint(uint32 sync_point) = 0;
  90
  91    protected:
  92     virtual ~SyncPointClient() {}
  93
  94     DISALLOW_COPY_AND_ASSIGN(SyncPointClient);
  95   };
  96
  97   // Call prior to CreateFrame to ensure validity of frame configuration. Called
  98   // automatically by VideoDecoderConfig::IsValidConfig().
  99   // TODO(scherkus): VideoDecoderConfig shouldn't call this method
 100   static bool IsValidConfig(VideoPixelFormat format,
 101                             StorageType storage_type,
 102                             const gfx::Size& coded_size,
 103                             const gfx::Rect& visible_rect,
 104                             const gfx::Size& natural_size);
 105
 106   // Creates a new YUV frame in system memory with given parameters (|format|
 107   // must be YUV). Buffers for the frame are allocated but not initialized. The
 108   // caller most not make assumptions about the actual underlying size(s), but
 109   // check the returned VideoFrame instead.
 110   // TODO(mcasas): implement the RGB version of this factory method.
 111   static scoped_refptr<VideoFrame> CreateFrame(VideoPixelFormat format,
 112                                                const gfx::Size& coded_size,
 113                                                const gfx::Rect& visible_rect,
 114                                                const gfx::Size& natural_size,
 115                                                base::TimeDelta timestamp);
 116
 117   // Wraps a native texture of the given parameters with a VideoFrame.
 118   // The backing of the VideoFrame is held in the mailbox held by
 119   // |mailbox_holder|, and |mailbox_holder_release_cb| will be called with
 120   // a syncpoint as the argument when the VideoFrame is to be destroyed.
 121   static scoped_refptr<VideoFrame> WrapNativeTexture(
 122       VideoPixelFormat format,
 123       const gpu::MailboxHolder& mailbox_holder,
 124       const ReleaseMailboxCB& mailbox_holder_release_cb,
 125       const gfx::Size& coded_size,
 126       const gfx::Rect& visible_rect,
 127       const gfx::Size& natural_size,
 128       base::TimeDelta timestamp);
 129
 130   // Wraps a set of native textures representing YUV data with a VideoFrame.
 131   // |mailbox_holders_release_cb| will be called with a syncpoint as the
 132   // argument when the VideoFrame is to be destroyed.
 133   static scoped_refptr<VideoFrame> WrapYUV420NativeTextures(
 134       const gpu::MailboxHolder& y_mailbox_holder,
 135       const gpu::MailboxHolder& u_mailbox_holder,
 136       const gpu::MailboxHolder& v_mailbox_holder,
 137       const ReleaseMailboxCB& mailbox_holders_release_cb,
 138       const gfx::Size& coded_size,
 139       const gfx::Rect& visible_rect,
 140       const gfx::Size& natural_size,
 141       base::TimeDelta timestamp);
 142
 143   // Wraps packed image data residing in a memory buffer with a VideoFrame.
 144   // The image data resides in |data| and is assumed to be packed tightly in a
 145   // buffer of logical dimensions |coded_size| with the appropriate bit depth
 146   // and plane count as given by |format|. Returns NULL on failure.
 147   static scoped_refptr<VideoFrame> WrapExternalData(
 148       VideoPixelFormat format,
 149       const gfx::Size& coded_size,
 150       const gfx::Rect& visible_rect,
 151       const gfx::Size& natural_size,
 152       uint8* data,
 153       size_t data_size,
 154       base::TimeDelta timestamp);
 155
 156   // Same as WrapExternalData() with SharedMemoryHandle and its offset.
 157   static scoped_refptr<VideoFrame> WrapExternalSharedMemory(
 158       VideoPixelFormat format,
 159       const gfx::Size& coded_size,
 160       const gfx::Rect& visible_rect,
 161       const gfx::Size& natural_size,
 162       uint8* data,
 163       size_t data_size,
 164       base::SharedMemoryHandle handle,
 165       size_t shared_memory_offset,
 166       base::TimeDelta timestamp);
 167
 168   // Wraps external YUV data of the given parameters with a VideoFrame.
 169   // The returned VideoFrame does not own the data passed in.
 170   static scoped_refptr<VideoFrame> WrapExternalYuvData(
 171       VideoPixelFormat format,
 172       const gfx::Size& coded_size,
 173       const gfx::Rect& visible_rect,
 174       const gfx::Size& natural_size,
 175       int32 y_stride,
 176       int32 u_stride,
 177       int32 v_stride,
 178       uint8* y_data,
 179       uint8* u_data,
 180       uint8* v_data,
 181       base::TimeDelta timestamp);
 182
 183 #if defined(OS_LINUX)
 184   // Wraps provided dmabufs
 185   // (https://www.kernel.org/doc/Documentation/dma-buf-sharing.txt) with a
 186   // VideoFrame. The dmabuf fds are dup()ed on creation, so that the VideoFrame
 187   // retains a reference to them, and are automatically close()d on destruction,
 188   // dropping the reference. The caller may safely close() its reference after
 189   // calling WrapExternalDmabufs().
 190   // The image data is only accessible via dmabuf fds, which are usually passed
 191   // directly to a hardware device and/or to another process, or can also be
 192   // mapped via mmap() for CPU access.
 193   // Returns NULL on failure.
 194   static scoped_refptr<VideoFrame> WrapExternalDmabufs(
 195       VideoPixelFormat format,
 196       const gfx::Size& coded_size,
 197       const gfx::Rect& visible_rect,
 198       const gfx::Size& natural_size,
 199       const std::vector<int>& dmabuf_fds,
 200       base::TimeDelta timestamp);
 201 #endif
 202
 203 #if defined(OS_MACOSX)
 204   // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
 205   // retained for the lifetime of the VideoFrame and released upon destruction.
 206   // The image data is only accessible via the pixel buffer, which could be
 207   // backed by an IOSurface from another process. All the attributes of the
 208   // VideoFrame are derived from the pixel buffer, with the exception of the
 209   // timestamp. If information is missing or is incompatible (for example, a
 210   // pixel format that has no VideoFrame match), NULL is returned.
 211   // http://crbug.com/401308
 212   static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
 213       CVPixelBufferRef cv_pixel_buffer,
 214       base::TimeDelta timestamp);
 215 #endif
 216
 217   // Wraps |frame|. |visible_rect| must be a sub rect within
 218   // frame->visible_rect().
 219   static scoped_refptr<VideoFrame> WrapVideoFrame(
 220       const scoped_refptr<VideoFrame>& frame,
 221       const gfx::Rect& visible_rect,
 222       const gfx::Size& natural_size);
 223
 224   // Creates a frame which indicates end-of-stream.
 225   static scoped_refptr<VideoFrame> CreateEOSFrame();
 226
 227   // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
 228   static scoped_refptr<VideoFrame> CreateColorFrame(
 229       const gfx::Size& size,
 230       uint8 y, uint8 u, uint8 v,
 231       base::TimeDelta timestamp);
 232
 233   // Allocates YV12 frame based on |size|, and sets its data to the YUV
 234   // equivalent of RGB(0,0,0).
 235   static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
 236
 237   // Allocates YV12A frame based on |size|, and sets its data to the YUVA
 238   // equivalent of RGBA(0,0,0,0).
 239   static scoped_refptr<VideoFrame> CreateTransparentFrame(
 240       const gfx::Size& size);
 241
 242 #if defined(VIDEO_HOLE)
 243   // Allocates a hole frame.
 244   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 245 #endif  // defined(VIDEO_HOLE)
 246
 247   static size_t NumPlanes(VideoPixelFormat format);
 248
 249   // Returns the required allocation size for a (tightly packed) frame of the
 250   // given coded size and format.
 251   static size_t AllocationSize(VideoPixelFormat format,
 252                                const gfx::Size& coded_size);
 253
 254   // Returns the plane gfx::Size (in bytes) for a plane of the given coded size
 255   // and format.
 256   static gfx::Size PlaneSize(VideoPixelFormat format,
 257                              size_t plane,
 258                              const gfx::Size& coded_size);
 259
 260   // Returns horizontal bits per pixel for given |plane| and |format|.
 261   static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane);
 262
 263   // Returns bits per pixel for given |plane| and |format|.
 264   static int PlaneBitsPerPixel(VideoPixelFormat format, size_t plane);
 265
 266   // Returns the number of bytes per row for the given plane, format, and width.
 267   // The width may be aligned to format requirements.
 268   static size_t RowBytes(size_t plane, VideoPixelFormat format, int width);
 269
 270   // Returns the number of rows for the given plane, format, and height.
 271   // The height may be aligned to format requirements.
 272   static size_t Rows(size_t plane, VideoPixelFormat format, int height);
 273
 274   // Returns the number of columns for the given plane, format, and width.
 275   // The width may be aligned to format requirements.
 276   static size_t Columns(size_t plane, VideoPixelFormat format, int width);
 277
 278   // Used to keep a running hash of seen frames.  Expects an initialized MD5
 279   // context.  Calls MD5Update with the context and the contents of the frame.
 280   static void HashFrameForTesting(base::MD5Context* context,
 281                                   const scoped_refptr<VideoFrame>& frame);
 282
 283   // Returns true if |frame| is accessible and mapped in the VideoFrame memory
 284   // space. If false, clients should refrain from accessing data(),
 285   // visible_data() etc.
 286   bool IsMappable() const;
 287
 288   // Returns true if |frame| has textures with any StorageType and should not be
 289   // accessed via data(), visible_data() etc.
 290   bool HasTextures() const;
 291
 292   VideoPixelFormat format() const { return format_; }
 293   StorageType storage_type() const { return storage_type_; }
 294
 295   const gfx::Size& coded_size() const { return coded_size_; }
 296   const gfx::Rect& visible_rect() const { return visible_rect_; }
 297   const gfx::Size& natural_size() const { return natural_size_; }
 298
 299   int stride(size_t plane) const;
 300
 301   // Returns the number of bytes per row and number of rows for a given plane.
 302   //
 303   // As opposed to stride(), row_bytes() refers to the bytes representing
 304   // frame data scanlines (coded_size.width() pixels, without stride padding).
 305   int row_bytes(size_t plane) const;
 306   int rows(size_t plane) const;
 307
 308   // Returns pointer to the buffer for a given plane, if this is an
 309   // IsMappable() frame type. The memory is owned by VideoFrame object and must
 310   // not be freed by the caller.
 311   const uint8* data(size_t plane) const;
 312   uint8* data(size_t plane);
 313
 314   // Returns pointer to the data in the visible region of the frame, for
 315   // IsMappable() storage types. The returned pointer is offsetted into the
 316   // plane buffer specified by visible_rect().origin(). Memory is owned by
 317   // VideoFrame object and must not be freed by the caller.
 318   const uint8* visible_data(size_t plane) const;
 319   uint8* visible_data(size_t plane);
 320
 321   // Returns a mailbox holder for a given texture.
 322   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
 323   // mailbox, the caller must wait for the included sync point.
 324   const gpu::MailboxHolder& mailbox_holder(size_t texture_index) const;
 325
 326   // Returns the shared-memory handle, if present
 327   base::SharedMemoryHandle shared_memory_handle() const;
 328
 329   // Returns the offset into the shared memory where the frame data begins.
 330   size_t shared_memory_offset() const;
 331
 332 #if defined(OS_LINUX)
 333   // Returns backing DmaBuf file descriptor for given |plane|, if present, or
 334   // -1 if not.
 335   // TODO(mcasas): Rename to DmabufFd() to comply with Style Guide.
 336   int dmabuf_fd(size_t plane) const;
 337
 338   // Duplicates internally the |fds_in|, overwriting the current ones. Returns
 339   // false if something goes wrong, and leaves all internal fds closed.
 340   bool DuplicateFileDescriptors(const std::vector<int>& fds_in);
 341 #endif
 342
 343   void AddSharedMemoryHandle(base::SharedMemoryHandle handle);
 344
 345 #if defined(OS_MACOSX)
 346   // Returns the backing CVPixelBuffer, if present.
 347   // TODO(mcasas): Rename to CvPixelBuffer() to comply with Style Guide.
 348   CVPixelBufferRef cv_pixel_buffer() const;
 349 #endif
 350
 351   // Adds a callback to be run when the VideoFrame is about to be destroyed.
 352   // The callback may be run from ANY THREAD, and so it is up to the client to
 353   // ensure thread safety.  Although read-only access to the members of this
 354   // VideoFrame is permitted while the callback executes (including
 355   // VideoFrameMetadata), clients should not assume the data pointers are
 356   // valid.
 357   void AddDestructionObserver(const base::Closure& callback);
 358
 359   // Returns a dictionary of optional metadata.  This contains information
 360   // associated with the frame that downstream clients might use for frame-level
 361   // logging, quality/performance optimizations, signaling, etc.
 362   //
 363   // TODO(miu): Move some of the "extra" members of VideoFrame (below) into
 364   // here as a later clean-up step.
 365   const VideoFrameMetadata* metadata() const { return &metadata_; }
 366   VideoFrameMetadata* metadata() { return &metadata_; }
 367
 368   base::TimeDelta timestamp() const { return timestamp_; }
 369   void set_timestamp(base::TimeDelta timestamp) {
 370     timestamp_ = timestamp;
 371   }
 372
 373   // It uses |client| to insert a new sync point and potentially waits on a
 374   // older sync point. The final sync point will be used to release this
 375   // VideoFrame.
 376   // This method is thread safe. Both blink and compositor threads can call it.
 377   void UpdateReleaseSyncPoint(SyncPointClient* client);
 378
 379  private:
 380   friend class base::RefCountedThreadSafe<VideoFrame>;
 381
 382   static scoped_refptr<VideoFrame> WrapExternalStorage(
 383       VideoPixelFormat format,
 384       StorageType storage_type,
 385       const gfx::Size& coded_size,
 386       const gfx::Rect& visible_rect,
 387       const gfx::Size& natural_size,
 388       uint8* data,
 389       size_t data_size,
 390       base::TimeDelta timestamp,
 391       base::SharedMemoryHandle handle,
 392       size_t data_offset);
 393
 394   // Clients must use the static factory/wrapping methods to create a new frame.
 395   VideoFrame(VideoPixelFormat format,
 396              StorageType storage_type,
 397              const gfx::Size& coded_size,
 398              const gfx::Rect& visible_rect,
 399              const gfx::Size& natural_size,
 400              base::TimeDelta timestamp);
 401   VideoFrame(VideoPixelFormat format,
 402              StorageType storage_type,
 403              const gfx::Size& coded_size,
 404              const gfx::Rect& visible_rect,
 405              const gfx::Size& natural_size,
 406              base::TimeDelta timestamp,
 407              base::SharedMemoryHandle handle,
 408              size_t shared_memory_offset);
 409   VideoFrame(VideoPixelFormat format,
 410              StorageType storage_type,
 411              const gfx::Size& coded_size,
 412              const gfx::Rect& visible_rect,
 413              const gfx::Size& natural_size,
 414              const gpu::MailboxHolder(&mailbox_holders)[kMaxPlanes],
 415              const ReleaseMailboxCB& mailbox_holder_release_cb,
 416              base::TimeDelta timestamp);
 417   virtual ~VideoFrame();
 418
 419   void AllocateYUV();
 420
 421   // Frame format.
 422   const VideoPixelFormat format_;
 423
 424   // Storage type for the different planes.
 425   StorageType storage_type_;  // TODO(mcasas): make const
 426
 427   // Width and height of the video frame, in pixels. This must include pixel
 428   // data for the whole image; i.e. for YUV formats with subsampled chroma
 429   // planes, in the case that the visible portion of the image does not line up
 430   // on a sample boundary, |coded_size_| must be rounded up appropriately and
 431   // the pixel data provided for the odd pixels.
 432   const gfx::Size coded_size_;
 433
 434   // Width, height, and offsets of the visible portion of the video frame. Must
 435   // be a subrect of |coded_size_|. Can be odd with respect to the sample
 436   // boundaries, e.g. for formats with subsampled chroma.
 437   const gfx::Rect visible_rect_;
 438
 439   // Width and height of the visible portion of the video frame
 440   // (|visible_rect_.size()|) with aspect ratio taken into account.
 441   const gfx::Size natural_size_;
 442
 443   // Array of strides for each plane, typically greater or equal to the width
 444   // of the surface divided by the horizontal sampling period.  Note that
 445   // strides can be negative.
 446   int32 strides_[kMaxPlanes];
 447
 448   // Array of data pointers to each plane.
 449   // TODO(mcasas): we don't know on ctor if we own |data_| or not. After
 450   // refactoring VideoFrame, change to scoped_ptr<uint8, AlignedFreeDeleter>.
 451   uint8* data_[kMaxPlanes];
 452
 453   // Native texture mailboxes, if this is a IsTexture() frame.
 454   gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
 455   ReleaseMailboxCB mailbox_holders_release_cb_;
 456
 457   // Shared memory handle and associated offset inside it, if this frame is
 458   // a STORAGE_SHMEM one.
 459   base::SharedMemoryHandle shared_memory_handle_;
 460   size_t shared_memory_offset_;
 461
 462 #if defined(OS_LINUX)
 463   // Dmabufs for each plane. If set, this frame has DmaBuf backing in some way.
 464   base::ScopedFD dmabuf_fds_[kMaxPlanes];
 465 #endif
 466
 467 #if defined(OS_MACOSX)
 468   // CVPixelBuffer, if this frame is wrapping one.
 469   base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
 470 #endif
 471
 472   std::vector<base::Closure> done_callbacks_;
 473
 474   base::TimeDelta timestamp_;
 475
 476   base::Lock release_sync_point_lock_;
 477   uint32 release_sync_point_;
 478
 479   VideoFrameMetadata metadata_;
 480
 481   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 482 };
 483
 484 }  // namespace media
 485
 486 #endif  // MEDIA_BASE_VIDEO_FRAME_H_