media/base/video_frame.h

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
   6 #define MEDIA_BASE_VIDEO_FRAME_H_
   7
   8 #include <vector>
   9
  10 #include "base/callback.h"
  11 #include "base/md5.h"
  12 #include "base/memory/shared_memory.h"
  13 #include "base/synchronization/lock.h"
  14 #include "media/base/buffers.h"
  15 #include "ui/gfx/rect.h"
  16 #include "ui/gfx/size.h"
  17
  18 #if defined(OS_MACOSX)
  19 #include <CoreVideo/CVPixelBuffer.h>
  20 #include "base/mac/scoped_cftyperef.h"
  21 #endif
  22
  23 class SkBitmap;
  24
  25 namespace gpu {
  26 struct MailboxHolder;
  27 }  // namespace gpu
  28
  29 namespace media {
  30
  31 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  32  public:
  33   enum {
  34     kFrameSizeAlignment = 16,
  35     kFrameSizePadding = 16,
  36     kFrameAddressAlignment = 32
  37   };
  38
  39   enum {
  40     kMaxPlanes = 4,
  41
  42     kYPlane = 0,
  43     kUPlane = 1,
  44     kUVPlane = kUPlane,
  45     kVPlane = 2,
  46     kAPlane = 3,
  47   };
  48
  49   // Surface formats roughly based on FOURCC labels, see:
  50   // http://www.fourcc.org/rgb.php
  51   // http://www.fourcc.org/yuv.php
  52   // Logged to UMA, so never reuse values.
  53   enum Format {
  54     UNKNOWN = 0,  // Unknown format value.
  55     YV12 = 1,  // 12bpp YVU planar 1x1 Y, 2x2 VU samples
  56     YV16 = 2,  // 16bpp YVU planar 1x1 Y, 2x1 VU samples
  57     I420 = 3,  // 12bpp YVU planar 1x1 Y, 2x2 UV samples.
  58     YV12A = 4,  // 20bpp YUVA planar 1x1 Y, 2x2 VU, 1x1 A samples.
  59 #if defined(VIDEO_HOLE)
  60     HOLE = 5,  // Hole frame.
  61 #endif  // defined(VIDEO_HOLE)
  62     NATIVE_TEXTURE = 6,  // Native texture.  Pixel-format agnostic.
  63     YV12J = 7,  // JPEG color range version of YV12
  64     NV12 = 8,  // 12bpp 1x1 Y plane followed by an interleaved 2x2 UV plane.
  65     YV24 = 9,  // 24bpp YUV planar, no subsampling.
  66     FORMAT_MAX = YV24,  // Must always be equal to largest entry logged.
  67   };
  68
  69   // Returns the name of a Format as a string.
  70   static std::string FormatToString(Format format);
  71
  72   // Creates a new frame in system memory with given parameters. Buffers for
  73   // the frame are allocated but not initialized.
  74   static scoped_refptr<VideoFrame> CreateFrame(
  75       Format format,
  76       const gfx::Size& coded_size,
  77       const gfx::Rect& visible_rect,
  78       const gfx::Size& natural_size,
  79       base::TimeDelta timestamp);
  80
  81   // Call prior to CreateFrame to ensure validity of frame configuration. Called
  82   // automatically by VideoDecoderConfig::IsValidConfig().
  83   // TODO(scherkus): VideoDecoderConfig shouldn't call this method
  84   static bool IsValidConfig(Format format, const gfx::Size& coded_size,
  85                             const gfx::Rect& visible_rect,
  86                             const gfx::Size& natural_size);
  87
  88   // CB to write pixels from the texture backing this frame into the
  89   // |const SkBitmap&| parameter.
  90   typedef base::Callback<void(const SkBitmap&)> ReadPixelsCB;
  91
  92   // CB to be called on the mailbox backing this frame when the frame is
  93   // destroyed.
  94   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
  95
  96   // Wraps a native texture of the given parameters with a VideoFrame.  The
  97   // backing of the VideoFrame is held in the mailbox held by |mailbox_holder|,
  98   // and |mailbox_holder_release_cb| will be called with |mailbox_holder| as the
  99   // argument when the VideoFrame is to be destroyed.
 100   // |read_pixels_cb| may be used to do (slow!) readbacks from the
 101   // texture to main memory.
 102   static scoped_refptr<VideoFrame> WrapNativeTexture(
 103       scoped_ptr<gpu::MailboxHolder> mailbox_holder,
 104       const ReleaseMailboxCB& mailbox_holder_release_cb,
 105       const gfx::Size& coded_size,
 106       const gfx::Rect& visible_rect,
 107       const gfx::Size& natural_size,
 108       base::TimeDelta timestamp,
 109       const ReadPixelsCB& read_pixels_cb);
 110
 111 #if !defined(MEDIA_FOR_CAST_IOS)
 112   // Read pixels from the native texture backing |*this| and write
 113   // them to |pixels| as BGRA.  |pixels| must point to a buffer at
 114   // least as large as 4 * visible_rect().size().GetArea().
 115   void ReadPixelsFromNativeTexture(const SkBitmap& pixels);
 116 #endif
 117
 118   // Wraps packed image data residing in a memory buffer with a VideoFrame.
 119   // The image data resides in |data| and is assumed to be packed tightly in a
 120   // buffer of logical dimensions |coded_size| with the appropriate bit depth
 121   // and plane count as given by |format|.  The shared memory handle of the
 122   // backing allocation, if present, can be passed in with |handle|.  When the
 123   // frame is destroyed, |no_longer_needed_cb.Run()| will be called.
 124   // Returns NULL on failure.
 125   static scoped_refptr<VideoFrame> WrapExternalPackedMemory(
 126       Format format,
 127       const gfx::Size& coded_size,
 128       const gfx::Rect& visible_rect,
 129       const gfx::Size& natural_size,
 130       uint8* data,
 131       size_t data_size,
 132       base::SharedMemoryHandle handle,
 133       base::TimeDelta timestamp,
 134       const base::Closure& no_longer_needed_cb);
 135
 136 #if defined(OS_POSIX)
 137   // Wraps provided dmabufs
 138   // (https://www.kernel.org/doc/Documentation/dma-buf-sharing.txt) with a
 139   // VideoFrame. The dmabuf fds are dup()ed on creation, so that the VideoFrame
 140   // retains a reference to them, and are automatically close()d on destruction,
 141   // dropping the reference. The caller may safely close() its reference after
 142   // calling WrapExternalDmabufs().
 143   // The image data is only accessible via dmabuf fds, which are usually passed
 144   // directly to a hardware device and/or to another process, or can also be
 145   // mapped via mmap() for CPU access.
 146   // When the frame is destroyed, |no_longer_needed_cb.Run()| will be called.
 147   // Returns NULL on failure.
 148   static scoped_refptr<VideoFrame> WrapExternalDmabufs(
 149       Format format,
 150       const gfx::Size& coded_size,
 151       const gfx::Rect& visible_rect,
 152       const gfx::Size& natural_size,
 153       const std::vector<int> dmabuf_fds,
 154       base::TimeDelta timestamp,
 155       const base::Closure& no_longer_needed_cb);
 156 #endif
 157
 158 #if defined(OS_MACOSX)
 159   // Wraps a provided CVPixelBuffer with a VideoFrame. The pixel buffer is
 160   // retained for the lifetime of the VideoFrame and released upon destruction.
 161   // The image data is only accessible via the pixel buffer, which could be
 162   // backed by an IOSurface from another process. All the attributes of the
 163   // VideoFrame are derived from the pixel buffer, with the exception of the
 164   // timestamp. If information is missing or is incompatible (for example, a
 165   // pixel format that has no VideoFrame match), NULL is returned.
 166   // http://crbug.com/401308
 167   static scoped_refptr<VideoFrame> WrapCVPixelBuffer(
 168       CVPixelBufferRef cv_pixel_buffer,
 169       base::TimeDelta timestamp);
 170 #endif
 171
 172   // Wraps external YUV data of the given parameters with a VideoFrame.
 173   // The returned VideoFrame does not own the data passed in. When the frame
 174   // is destroyed |no_longer_needed_cb.Run()| will be called.
 175   // TODO(sheu): merge this into WrapExternalSharedMemory().
 176   // http://crbug.com/270217
 177   static scoped_refptr<VideoFrame> WrapExternalYuvData(
 178       Format format,
 179       const gfx::Size& coded_size,
 180       const gfx::Rect& visible_rect,
 181       const gfx::Size& natural_size,
 182       int32 y_stride,
 183       int32 u_stride,
 184       int32 v_stride,
 185       uint8* y_data,
 186       uint8* u_data,
 187       uint8* v_data,
 188       base::TimeDelta timestamp,
 189       const base::Closure& no_longer_needed_cb);
 190
 191   // Wraps |frame| and calls |no_longer_needed_cb| when the wrapper VideoFrame
 192   // gets destroyed. |visible_rect| must be a sub rect within
 193   // frame->visible_rect().
 194   static scoped_refptr<VideoFrame> WrapVideoFrame(
 195       const scoped_refptr<VideoFrame>& frame,
 196       const gfx::Rect& visible_rect,
 197       const gfx::Size& natural_size,
 198       const base::Closure& no_longer_needed_cb);
 199
 200   // Creates a frame which indicates end-of-stream.
 201   static scoped_refptr<VideoFrame> CreateEOSFrame();
 202
 203   // Allocates YV12 frame based on |size|, and sets its data to the YUV(y,u,v).
 204   static scoped_refptr<VideoFrame> CreateColorFrame(
 205       const gfx::Size& size,
 206       uint8 y, uint8 u, uint8 v,
 207       base::TimeDelta timestamp);
 208
 209   // Allocates YV12 frame based on |size|, and sets its data to the YUV
 210   // equivalent of RGB(0,0,0).
 211   static scoped_refptr<VideoFrame> CreateBlackFrame(const gfx::Size& size);
 212
 213   // Allocates YV12A frame based on |size|, and sets its data to the YUVA
 214   // equivalent of RGBA(0,0,0,0).
 215   static scoped_refptr<VideoFrame> CreateTransparentFrame(
 216       const gfx::Size& size);
 217
 218 #if defined(VIDEO_HOLE)
 219   // Allocates a hole frame.
 220   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 221 #endif  // defined(VIDEO_HOLE)
 222
 223   static size_t NumPlanes(Format format);
 224
 225   // Returns the required allocation size for a (tightly packed) frame of the
 226   // given coded size and format.
 227   static size_t AllocationSize(Format format, const gfx::Size& coded_size);
 228
 229   // Returns the plane size for a plane of the given coded size and format.
 230   static gfx::Size PlaneSize(Format format,
 231                              size_t plane,
 232                              const gfx::Size& coded_size);
 233
 234   // Returns the required allocation size for a (tightly packed) plane of the
 235   // given coded size and format.
 236   static size_t PlaneAllocationSize(Format format,
 237                                     size_t plane,
 238                                     const gfx::Size& coded_size);
 239
 240   // Returns horizontal bits per pixel for given |plane| and |format|.
 241   static int PlaneHorizontalBitsPerPixel(Format format, size_t plane);
 242
 243   // Returns the number of bytes per row for the given plane, format, and width.
 244   // The width may be aligned to format requirements.
 245   static size_t RowBytes(size_t plane, Format format, int width);
 246
 247   // Returns the number of rows for the given plane, format, and height.
 248   // The height may be aligned to format requirements.
 249   static size_t Rows(size_t plane, Format format, int height);
 250
 251   Format format() const { return format_; }
 252
 253   const gfx::Size& coded_size() const { return coded_size_; }
 254   const gfx::Rect& visible_rect() const { return visible_rect_; }
 255   const gfx::Size& natural_size() const { return natural_size_; }
 256
 257   int stride(size_t plane) const;
 258
 259   // Returns the number of bytes per row and number of rows for a given plane.
 260   //
 261   // As opposed to stride(), row_bytes() refers to the bytes representing
 262   // frame data scanlines (coded_size.width() pixels, without stride padding).
 263   int row_bytes(size_t plane) const;
 264   int rows(size_t plane) const;
 265
 266   // Returns pointer to the buffer for a given plane. The memory is owned by
 267   // VideoFrame object and must not be freed by the caller.
 268   uint8* data(size_t plane) const;
 269
 270   // Returns the mailbox holder of the native texture wrapped by this frame.
 271   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
 272   // mailbox, the caller must wait for the included sync point.
 273   const gpu::MailboxHolder* mailbox_holder() const;
 274
 275   // Returns the shared-memory handle, if present
 276   base::SharedMemoryHandle shared_memory_handle() const;
 277
 278 #if defined(OS_POSIX)
 279   // Returns backing dmabuf file descriptor for given |plane|, if present.
 280   int dmabuf_fd(size_t plane) const;
 281 #endif
 282
 283 #if defined(OS_MACOSX)
 284   // Returns the backing CVPixelBuffer, if present.
 285   CVPixelBufferRef cv_pixel_buffer() const;
 286 #endif
 287
 288   // Returns true if this VideoFrame represents the end of the stream.
 289   bool end_of_stream() const { return end_of_stream_; }
 290
 291   base::TimeDelta timestamp() const {
 292     return timestamp_;
 293   }
 294   void set_timestamp(const base::TimeDelta& timestamp) {
 295     timestamp_ = timestamp;
 296   }
 297
 298   class SyncPointClient {
 299    public:
 300     SyncPointClient() {}
 301     virtual uint32 InsertSyncPoint() = 0;
 302     virtual void WaitSyncPoint(uint32 sync_point) = 0;
 303
 304    protected:
 305     virtual ~SyncPointClient() {}
 306
 307     DISALLOW_COPY_AND_ASSIGN(SyncPointClient);
 308   };
 309   // It uses |client| to insert a new sync point and potentially waits on a
 310   // older sync point. The final sync point will be used to release this
 311   // VideoFrame.
 312   // This method is thread safe. Both blink and compositor threads can call it.
 313   void UpdateReleaseSyncPoint(SyncPointClient* client);
 314
 315   // Used to keep a running hash of seen frames.  Expects an initialized MD5
 316   // context.  Calls MD5Update with the context and the contents of the frame.
 317   void HashFrameForTesting(base::MD5Context* context);
 318
 319  private:
 320   friend class base::RefCountedThreadSafe<VideoFrame>;
 321
 322   // Returns true if |plane| is a valid plane number for the given format. This
 323   // can be used to DCHECK() plane parameters.
 324   static bool IsValidPlane(size_t plane, VideoFrame::Format format);
 325
 326   // Clients must use the static CreateFrame() method to create a new frame.
 327   VideoFrame(Format format,
 328              const gfx::Size& coded_size,
 329              const gfx::Rect& visible_rect,
 330              const gfx::Size& natural_size,
 331              scoped_ptr<gpu::MailboxHolder> mailbox_holder,
 332              base::TimeDelta timestamp,
 333              bool end_of_stream);
 334   virtual ~VideoFrame();
 335
 336   void AllocateYUV();
 337
 338   // Frame format.
 339   const Format format_;
 340
 341   // Width and height of the video frame, in pixels. This must include pixel
 342   // data for the whole image; i.e. for YUV formats with subsampled chroma
 343   // planes, in the case that the visible portion of the image does not line up
 344   // on a sample boundary, |coded_size_| must be rounded up appropriately and
 345   // the pixel data provided for the odd pixels.
 346   const gfx::Size coded_size_;
 347
 348   // Width, height, and offsets of the visible portion of the video frame. Must
 349   // be a subrect of |coded_size_|. Can be odd with respect to the sample
 350   // boundaries, e.g. for formats with subsampled chroma.
 351   const gfx::Rect visible_rect_;
 352
 353   // Width and height of the visible portion of the video frame
 354   // (|visible_rect_.size()|) with aspect ratio taken into account.
 355   const gfx::Size natural_size_;
 356
 357   // Array of strides for each plane, typically greater or equal to the width
 358   // of the surface divided by the horizontal sampling period.  Note that
 359   // strides can be negative.
 360   int32 strides_[kMaxPlanes];
 361
 362   // Array of data pointers to each plane.
 363   uint8* data_[kMaxPlanes];
 364
 365   // Native texture mailbox, if this is a NATIVE_TEXTURE frame.
 366   const scoped_ptr<gpu::MailboxHolder> mailbox_holder_;
 367   ReleaseMailboxCB mailbox_holder_release_cb_;
 368   ReadPixelsCB read_pixels_cb_;
 369
 370   // Shared memory handle, if this frame was allocated from shared memory.
 371   base::SharedMemoryHandle shared_memory_handle_;
 372
 373 #if defined(OS_POSIX)
 374   // Dmabufs for each plane, if this frame is wrapping memory
 375   // acquired via dmabuf.
 376   base::ScopedFD dmabuf_fds_[kMaxPlanes];
 377 #endif
 378
 379 #if defined(OS_MACOSX)
 380   // CVPixelBuffer, if this frame is wrapping one.
 381   base::ScopedCFTypeRef<CVPixelBufferRef> cv_pixel_buffer_;
 382 #endif
 383
 384   base::Closure no_longer_needed_cb_;
 385
 386   base::TimeDelta timestamp_;
 387
 388   base::Lock release_sync_point_lock_;
 389   uint32 release_sync_point_;
 390
 391   const bool end_of_stream_;
 392
 393   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 394 };
 395
 396 }  // namespace media
 397
 398 #endif  // MEDIA_BASE_VIDEO_FRAME_H_