media/video/gpu_memory_buffer_video_frame_pool.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/video/gpu_memory_buffer_video_frame_pool.h"
   6
   7 #include <GLES2/gl2.h>
   8 #include <GLES2/gl2ext.h>
   9
  10 #include <list>
  11 #include <utility>
  12
  13 #include "base/bind.h"
  14 #include "base/containers/stack_container.h"
  15 #include "base/location.h"
  16 #include "base/memory/linked_ptr.h"
  17 #include "base/single_thread_task_runner.h"
  18 #include "base/trace_event/trace_event.h"
  19 #include "gpu/command_buffer/client/gles2_interface.h"
  20 #include "media/renderers/gpu_video_accelerator_factories.h"
  21
  22 namespace media {
  23
  24 // Implementation of a pool of GpuMemoryBuffers used to back VideoFrames.
  25 class GpuMemoryBufferVideoFramePool::PoolImpl
  26     : public base::RefCountedThreadSafe<
  27           GpuMemoryBufferVideoFramePool::PoolImpl> {
  28  public:
  29   // |task_runner| is associated to the thread where the context of
  30   // GLES2Interface returned by |gpu_factories| lives.
  31   // |gpu_factories| is an interface to GPU related operation and can be
  32   // null.
  33   PoolImpl(const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
  34            const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories)
  35       : task_runner_(task_runner),
  36         gpu_factories_(gpu_factories),
  37         texture_target_(gpu_factories_ ? gpu_factories_->ImageTextureTarget()
  38                                        : GL_TEXTURE_2D) {}
  39
  40   // Takes a software VideoFrame and returns a VideoFrame backed by native
  41   // textures if possible.
  42   // The data contained in video_frame is copied into the returned frame.
  43   scoped_refptr<VideoFrame> CreateHardwareFrame(
  44       const scoped_refptr<VideoFrame>& video_frame);
  45
  46  private:
  47   friend class base::RefCountedThreadSafe<
  48       GpuMemoryBufferVideoFramePool::PoolImpl>;
  49   ~PoolImpl();
  50
  51   // Resource to represent a plane.
  52   struct PlaneResource {
  53     gfx::Size size;
  54     scoped_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer;
  55     unsigned texture_id = 0u;
  56     unsigned image_id = 0u;
  57     gpu::Mailbox mailbox;
  58   };
  59
  60   // All the resources needed to compose a frame.
  61   struct FrameResources {
  62     FrameResources(VideoPixelFormat format, const gfx::Size& size)
  63         : format(format), size(size) {}
  64     bool in_use = true;
  65     VideoPixelFormat format;
  66     gfx::Size size;
  67     PlaneResource plane_resources[VideoFrame::kMaxPlanes];
  68   };
  69
  70   // Return true if |resources| can be used to represent a frame for
  71   // specific |format| and |size|.
  72   static bool AreFrameResourcesCompatible(const FrameResources* resources,
  73                                           const gfx::Size& size,
  74                                           VideoPixelFormat format) {
  75     return size == resources->size && format == resources->format;
  76   }
  77
  78   // Get the resources needed for a frame out of the pool, or create them if
  79   // necessary.
  80   // This also drops the LRU resources that can't be reuse for this frame.
  81   FrameResources* GetOrCreateFrameResources(const gfx::Size& size,
  82                                             VideoPixelFormat format);
  83
  84   // Callback called when a VideoFrame generated with GetFrameResources is no
  85   // longer referenced.
  86   // This could be called by any thread.
  87   void MailboxHoldersReleased(FrameResources* frame_resources,
  88                               uint32 sync_point);
  89
  90   // Return frame resources to the pool. This has to be called on the thread
  91   // where |task_runner| is current.
  92   void ReturnFrameResources(FrameResources* frame_resources);
  93
  94   // Delete resources. This has to be called on the thread where |task_runner|
  95   // is current.
  96   static void DeleteFrameResources(
  97       const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories,
  98       FrameResources* frame_resources);
  99
 100   scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
 101   scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories_;
 102
 103   // Pool of resources.
 104   std::list<FrameResources*> resources_pool_;
 105
 106   const unsigned texture_target_;
 107   DISALLOW_COPY_AND_ASSIGN(PoolImpl);
 108 };
 109
 110 namespace {
 111
 112 // Copy a buffer info a GpuMemoryBuffer.
 113 // |bytes_per_row| is expected to be less or equal than the strides of the two
 114 // buffers.
 115 void CopyPlaneToGpuMemoryBuffer(int rows,
 116                                 int bytes_per_row,
 117                                 const uint8* source,
 118                                 int source_stride,
 119                                 gfx::GpuMemoryBuffer* buffer) {
 120   TRACE_EVENT2("media", "CopyPlaneToGpuMemoryBuffer", "bytes_per_row",
 121                bytes_per_row, "rows", rows);
 122
 123   DCHECK(buffer);
 124   DCHECK(source);
 125   void* data = nullptr;
 126   CHECK(buffer->Map(&data));
 127   uint8* mapped_buffer = static_cast<uint8*>(data);
 128   int dest_stride = 0;
 129   buffer->GetStride(&dest_stride);
 130   DCHECK_NE(dest_stride, 0);
 131   DCHECK_LE(bytes_per_row, std::abs(dest_stride));
 132   DCHECK_LE(bytes_per_row, source_stride);
 133   for (int row = 0; row < rows; ++row) {
 134     memcpy(mapped_buffer + dest_stride * row, source + source_stride * row,
 135            bytes_per_row);
 136   }
 137   buffer->Unmap();
 138 }
 139
 140 }  // unnamed namespace
 141
 142 // Creates a VideoFrame backed by native textures starting from a software
 143 // VideoFrame.
 144 // The data contained in video_frame is copied into the returned VideoFrame.
 145 scoped_refptr<VideoFrame>
 146 GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
 147     const scoped_refptr<VideoFrame>& video_frame) {
 148   if (!gpu_factories_)
 149     return video_frame;
 150
 151   if (!gpu_factories_->IsTextureRGSupported())
 152     return video_frame;
 153
 154   gpu::gles2::GLES2Interface* gles2 = gpu_factories_->GetGLES2Interface();
 155   if (!gles2)
 156     return video_frame;
 157
 158   VideoPixelFormat format = video_frame->format();
 159   size_t planes = VideoFrame::NumPlanes(format);
 160   DCHECK(video_frame->visible_rect().origin().IsOrigin());
 161   gfx::Size size = video_frame->visible_rect().size();
 162   gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];
 163
 164   // Acquire resources. Incompatible ones will be dropped from the pool.
 165   FrameResources* frame_resources = GetOrCreateFrameResources(size, format);
 166
 167   // Set up the planes copying data into it and creating the mailboxes needed
 168   // to refer to the textures.
 169   for (size_t i = 0; i < planes; ++i) {
 170     PlaneResource& plane_resource = frame_resources->plane_resources[i];
 171     CopyPlaneToGpuMemoryBuffer(VideoFrame::Rows(i, format, size.height()),
 172                                VideoFrame::RowBytes(i, format, size.width()),
 173                                video_frame->data(i), video_frame->stride(i),
 174                                plane_resource.gpu_memory_buffer.get());
 175
 176     // Bind the texture and create or rebind the image.
 177     gles2->BindTexture(texture_target_, plane_resource.texture_id);
 178
 179     if (plane_resource.gpu_memory_buffer && !plane_resource.image_id) {
 180       const size_t width = VideoFrame::Columns(i, format, size.width());
 181       const size_t height = VideoFrame::Rows(i, format, size.height());
 182       plane_resource.image_id = gles2->CreateImageCHROMIUM(
 183           plane_resource.gpu_memory_buffer->AsClientBuffer(), width, height,
 184           GL_R8_EXT);
 185     } else {
 186       gles2->ReleaseTexImage2DCHROMIUM(texture_target_,
 187                                        plane_resource.image_id);
 188     }
 189     gles2->BindTexImage2DCHROMIUM(texture_target_, plane_resource.image_id);
 190     mailbox_holders[i] =
 191         gpu::MailboxHolder(plane_resource.mailbox, texture_target_, 0);
 192   }
 193
 194   // Insert a sync_point, this is needed to make sure that the textures the
 195   // mailboxes refer to will be used only after all the previous commands posted
 196   // in the command buffer have been processed.
 197   unsigned sync_point = gles2->InsertSyncPointCHROMIUM();
 198   for (size_t i = 0; i < planes; ++i) {
 199     mailbox_holders[i].sync_point = sync_point;
 200   }
 201
 202   // Create the VideoFrame backed by native textures.
 203   scoped_refptr<VideoFrame> frame = VideoFrame::WrapYUV420NativeTextures(
 204       mailbox_holders[VideoFrame::kYPlane],
 205       mailbox_holders[VideoFrame::kUPlane],
 206       mailbox_holders[VideoFrame::kVPlane],
 207       base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources),
 208       size, video_frame->visible_rect(), video_frame->natural_size(),
 209       video_frame->timestamp());
 210   if (video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY))
 211     frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, true);
 212   return frame;
 213 }
 214
 215 // Destroy all the resources posting one task per FrameResources
 216 // to the |task_runner_|.
 217 GpuMemoryBufferVideoFramePool::PoolImpl::~PoolImpl() {
 218   // Delete all the resources on the media thread.
 219   while (!resources_pool_.empty()) {
 220     FrameResources* frame_resources = resources_pool_.front();
 221     resources_pool_.pop_front();
 222     task_runner_->PostTask(
 223         FROM_HERE, base::Bind(&PoolImpl::DeleteFrameResources, gpu_factories_,
 224                               base::Owned(frame_resources)));
 225   }
 226 }
 227
 228 // Tries to find the resources in the pool or create them.
 229 // Incompatible resources will be dropped.
 230 GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources*
 231 GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
 232     const gfx::Size& size,
 233     VideoPixelFormat format) {
 234   DCHECK(task_runner_->BelongsToCurrentThread());
 235
 236   auto it = resources_pool_.begin();
 237   while (it != resources_pool_.end()) {
 238     FrameResources* frame_resources = *it;
 239     if (!frame_resources->in_use) {
 240       if (AreFrameResourcesCompatible(frame_resources, size, format)) {
 241         frame_resources->in_use = true;
 242         return frame_resources;
 243       } else {
 244         resources_pool_.erase(it++);
 245         DeleteFrameResources(gpu_factories_, frame_resources);
 246         delete frame_resources;
 247       }
 248     } else {
 249       it++;
 250     }
 251   }
 252
 253   // Create the resources.
 254   gpu::gles2::GLES2Interface* gles2 = gpu_factories_->GetGLES2Interface();
 255   DCHECK(gles2);
 256   gles2->ActiveTexture(GL_TEXTURE0);
 257   size_t planes = VideoFrame::NumPlanes(format);
 258   FrameResources* frame_resources = new FrameResources(format, size);
 259   resources_pool_.push_back(frame_resources);
 260   for (size_t i = 0; i < planes; ++i) {
 261     PlaneResource& plane_resource = frame_resources->plane_resources[i];
 262     const size_t width = VideoFrame::Columns(i, format, size.width());
 263     const size_t height = VideoFrame::Rows(i, format, size.height());
 264     const gfx::Size plane_size(width, height);
 265     plane_resource.gpu_memory_buffer = gpu_factories_->AllocateGpuMemoryBuffer(
 266         plane_size, gfx::BufferFormat::R_8, gfx::BufferUsage::MAP);
 267
 268     gles2->GenTextures(1, &plane_resource.texture_id);
 269     gles2->BindTexture(texture_target_, plane_resource.texture_id);
 270     gles2->TexParameteri(texture_target_, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 271     gles2->TexParameteri(texture_target_, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 272     gles2->TexParameteri(texture_target_, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 273     gles2->TexParameteri(texture_target_, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 274     gles2->GenMailboxCHROMIUM(plane_resource.mailbox.name);
 275     gles2->ProduceTextureCHROMIUM(texture_target_, plane_resource.mailbox.name);
 276   }
 277   return frame_resources;
 278 }
 279
 280 // static
 281 void GpuMemoryBufferVideoFramePool::PoolImpl::DeleteFrameResources(
 282     const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories,
 283     FrameResources* frame_resources) {
 284   // TODO(dcastagna): As soon as the context lost is dealt with in media,
 285   // make sure that we won't execute this callback (use a weak pointer to
 286   // the old context).
 287   gpu::gles2::GLES2Interface* gles2 = gpu_factories->GetGLES2Interface();
 288   if (!gles2)
 289     return;
 290
 291   for (PlaneResource& plane_resource : frame_resources->plane_resources) {
 292     if (plane_resource.image_id)
 293       gles2->DestroyImageCHROMIUM(plane_resource.image_id);
 294     if (plane_resource.texture_id)
 295       gles2->DeleteTextures(1, &plane_resource.texture_id);
 296   }
 297 }
 298
 299 // Called when a VideoFrame is no longer references.
 300 void GpuMemoryBufferVideoFramePool::PoolImpl::MailboxHoldersReleased(
 301     FrameResources* frame_resources,
 302     uint32 sync_point) {
 303   // Return the resource on the media thread.
 304   task_runner_->PostTask(FROM_HERE, base::Bind(&PoolImpl::ReturnFrameResources,
 305                                                this, frame_resources));
 306 }
 307
 308 // Put back the resoruces in the pool.
 309 void GpuMemoryBufferVideoFramePool::PoolImpl::ReturnFrameResources(
 310     FrameResources* frame_resources) {
 311   DCHECK(task_runner_->BelongsToCurrentThread());
 312
 313   auto it = std::find(resources_pool_.begin(), resources_pool_.end(),
 314                       frame_resources);
 315   DCHECK(it != resources_pool_.end());
 316   // We want the pool to behave in a FIFO way.
 317   // This minimizes the chances of locking the buffer that might be
 318   // still needed for drawing.
 319   std::swap(*it, resources_pool_.back());
 320   frame_resources->in_use = false;
 321 }
 322
 323 GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool(
 324     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
 325     const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories)
 326     : pool_impl_(new PoolImpl(task_runner, gpu_factories)) {
 327 }
 328
 329 GpuMemoryBufferVideoFramePool::~GpuMemoryBufferVideoFramePool() {
 330 }
 331
 332 scoped_refptr<VideoFrame>
 333 GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame(
 334     const scoped_refptr<VideoFrame>& video_frame) {
 335   switch (video_frame->format()) {
 336     // Supported cases.
 337     case PIXEL_FORMAT_YV12:
 338     case PIXEL_FORMAT_I420:
 339       return pool_impl_->CreateHardwareFrame(video_frame);
 340     // Unsupported cases.
 341     case PIXEL_FORMAT_YV12A:
 342     case PIXEL_FORMAT_YV16:
 343     case PIXEL_FORMAT_YV24:
 344 #if defined(OS_MACOSX) || defined(OS_CHROMEOS)
 345     case PIXEL_FORMAT_NV12:
 346 #endif
 347     case PIXEL_FORMAT_ARGB:
 348     case PIXEL_FORMAT_XRGB:
 349     case PIXEL_FORMAT_UNKNOWN:
 350       break;
 351   }
 352   return video_frame;
 353 }
 354
 355 }  // namespace media