media/video/gpu_memory_buffer_video_frame_pool.cc

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "media/video/gpu_memory_buffer_video_frame_pool.h"
   6
   7 #include <GLES2/gl2.h>
   8 #include <GLES2/gl2ext.h>
   9
  10 #include <algorithm>
  11 #include <list>
  12 #include <utility>
  13
  14 #include "base/barrier_closure.h"
  15 #include "base/bind.h"
  16 #include "base/containers/stack_container.h"
  17 #include "base/location.h"
  18 #include "base/memory/linked_ptr.h"
  19 #include "base/trace_event/trace_event.h"
  20 #include "gpu/command_buffer/client/gles2_interface.h"
  21 #include "media/renderers/gpu_video_accelerator_factories.h"
  22
  23 namespace media {
  24
  25 // Implementation of a pool of GpuMemoryBuffers used to back VideoFrames.
  26 class GpuMemoryBufferVideoFramePool::PoolImpl
  27     : public base::RefCountedThreadSafe<
  28           GpuMemoryBufferVideoFramePool::PoolImpl> {
  29  public:
  30   // |media_task_runner| is the media task runner associated with the
  31   // GL context provided by |gpu_factories|
  32   // |worker_task_runner| is a task runner used to asynchronously copy
  33   // video frame's planes.
  34   // |gpu_factories| is an interface to GPU related operation and can be
  35   // null if a GL context is not available.
  36   PoolImpl(const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
  37            const scoped_refptr<base::TaskRunner>& worker_task_runner,
  38            const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories)
  39       : media_task_runner_(media_task_runner),
  40         worker_task_runner_(worker_task_runner),
  41         gpu_factories_(gpu_factories),
  42         texture_target_(gpu_factories ? gpu_factories->ImageTextureTarget()
  43                                       : GL_TEXTURE_2D) {
  44     DCHECK(media_task_runner_);
  45     DCHECK(worker_task_runner_);
  46   }
  47
  48   // Takes a software VideoFrame and calls |frame_ready_cb| with a VideoFrame
  49   // backed by native textures if possible.
  50   // The data contained in video_frame is copied into the returned frame
  51   // asynchronously posting tasks to |worker_task_runner_|, while
  52   // |frame_ready_cb| will be called on |media_task_runner_| once all the data
  53   // has been copied.
  54   void CreateHardwareFrame(const scoped_refptr<VideoFrame>& video_frame,
  55                            const FrameReadyCB& cb);
  56
  57  private:
  58   friend class base::RefCountedThreadSafe<
  59       GpuMemoryBufferVideoFramePool::PoolImpl>;
  60   ~PoolImpl();
  61
  62   // Resource to represent a plane.
  63   struct PlaneResource {
  64     gfx::Size size;
  65     scoped_ptr<gfx::GpuMemoryBuffer> gpu_memory_buffer;
  66     unsigned texture_id = 0u;
  67     unsigned image_id = 0u;
  68     gpu::Mailbox mailbox;
  69   };
  70
  71   // All the resources needed to compose a frame.
  72   struct FrameResources {
  73     FrameResources(VideoPixelFormat format, const gfx::Size& size)
  74         : format(format), size(size) {}
  75     bool in_use = true;
  76     VideoPixelFormat format;
  77     gfx::Size size;
  78     PlaneResource plane_resources[VideoFrame::kMaxPlanes];
  79   };
  80
  81   // Copy |video_frame| data into |frame_resouces|
  82   // and calls |done| when done.
  83   void CopyVideoFrameToGpuMemoryBuffers(
  84       const scoped_refptr<VideoFrame>& video_frame,
  85       FrameResources* frame_resources,
  86       const FrameReadyCB& frame_ready_cb);
  87
  88   // Called when all the data has been copied.
  89   void OnCopiesDone(const scoped_refptr<VideoFrame>& video_frame,
  90                     FrameResources* frame_resources,
  91                     const FrameReadyCB& frame_ready_cb);
  92
  93   // Prepares GL resources, mailboxes and calls |frame_ready_cb| with the new
  94   // VideoFrame.
  95   // This has to be run on |media_task_runner_| where |frame_ready_cb| will also
  96   // be run.
  97   void BindAndCreateMailboxesHardwareFrameResources(
  98       const scoped_refptr<VideoFrame>& video_frame,
  99       FrameResources* frame_resources,
 100       const FrameReadyCB& frame_ready_cb);
 101
 102   // Return true if |resources| can be used to represent a frame for
 103   // specific |format| and |size|.
 104   static bool AreFrameResourcesCompatible(const FrameResources* resources,
 105                                           const gfx::Size& size,
 106                                           VideoPixelFormat format) {
 107     return size == resources->size && format == resources->format;
 108   }
 109
 110   // Get the resources needed for a frame out of the pool, or create them if
 111   // necessary.
 112   // This also drops the LRU resources that can't be reuse for this frame.
 113   FrameResources* GetOrCreateFrameResources(const gfx::Size& size,
 114                                             VideoPixelFormat format);
 115
 116   // Callback called when a VideoFrame generated with GetFrameResources is no
 117   // longer referenced.
 118   // This could be called by any thread.
 119   void MailboxHoldersReleased(FrameResources* frame_resources,
 120                               uint32 sync_point);
 121
 122   // Return frame resources to the pool. This has to be called on the thread
 123   // where |media_task_runner_| is current.
 124   void ReturnFrameResources(FrameResources* frame_resources);
 125
 126   // Delete resources. This has to be called on the thread where |task_runner|
 127   // is current.
 128   static void DeleteFrameResources(
 129       const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories,
 130       FrameResources* frame_resources);
 131
 132   // Task runner associated to the GL context provided by |gpu_factories_|.
 133   scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
 134   // Task runner used to asynchronously copy planes.
 135   scoped_refptr<base::TaskRunner> worker_task_runner_;
 136
 137   // Interface to GPU related operations.
 138   scoped_refptr<GpuVideoAcceleratorFactories> gpu_factories_;
 139
 140   // Pool of resources.
 141   std::list<FrameResources*> resources_pool_;
 142
 143   const unsigned texture_target_;
 144   DISALLOW_COPY_AND_ASSIGN(PoolImpl);
 145 };
 146
 147 namespace {
 148
 149 // VideoFrame copies to GpuMemoryBuffers will be split in |kBytesPerCopyTarget|
 150 // bytes copies and run in parallel.
 151 const size_t kBytesPerCopyTarget = 1024 * 1024;  // 1MB
 152
 153 void CopyRowsToBuffer(int first_row,
 154                       int rows,
 155                       int bytes_per_row,
 156                       const uint8* source,
 157                       int source_stride,
 158                       uint8* output,
 159                       int dest_stride,
 160                       const base::Closure& done) {
 161   TRACE_EVENT2("media", "CopyRowsToBuffer", "bytes_per_row", bytes_per_row,
 162                "rows", rows);
 163   DCHECK_NE(dest_stride, 0);
 164   DCHECK_LE(bytes_per_row, std::abs(dest_stride));
 165   DCHECK_LE(bytes_per_row, source_stride);
 166   for (int row = first_row; row < first_row + rows; ++row) {
 167     memcpy(output + dest_stride * row, source + source_stride * row,
 168            bytes_per_row);
 169   }
 170   done.Run();
 171 }
 172
 173 }  // unnamed namespace
 174
 175 // Creates a VideoFrame backed by native textures starting from a software
 176 // VideoFrame.
 177 // The data contained in |video_frame| is copied into the VideoFrame passed to
 178 // |frame_ready_cb|.
 179 // This has to be called on the thread where |media_task_runner_| is current.
 180 void GpuMemoryBufferVideoFramePool::PoolImpl::CreateHardwareFrame(
 181     const scoped_refptr<VideoFrame>& video_frame,
 182     const FrameReadyCB& frame_ready_cb) {
 183   DCHECK(media_task_runner_->BelongsToCurrentThread());
 184   if (!gpu_factories_ || !gpu_factories_->IsTextureRGSupported()) {
 185     frame_ready_cb.Run(video_frame);
 186     return;
 187   }
 188   switch (video_frame->format()) {
 189     // Supported cases.
 190     case PIXEL_FORMAT_YV12:
 191     case PIXEL_FORMAT_I420:
 192       break;
 193     // Unsupported cases.
 194     case PIXEL_FORMAT_YV12A:
 195     case PIXEL_FORMAT_YV16:
 196     case PIXEL_FORMAT_YV24:
 197     case PIXEL_FORMAT_NV12:
 198     case PIXEL_FORMAT_ARGB:
 199     case PIXEL_FORMAT_XRGB:
 200     case PIXEL_FORMAT_UYVY:
 201     case PIXEL_FORMAT_UNKNOWN:
 202       frame_ready_cb.Run(video_frame);
 203       return;
 204   }
 205
 206   VideoPixelFormat format = video_frame->format();
 207   DCHECK(video_frame->visible_rect().origin().IsOrigin());
 208   const gfx::Size size = video_frame->visible_rect().size();
 209
 210   // Acquire resources. Incompatible ones will be dropped from the pool.
 211   FrameResources* frame_resources = GetOrCreateFrameResources(size, format);
 212   if (!frame_resources) {
 213     frame_ready_cb.Run(video_frame);
 214     return;
 215   }
 216
 217   worker_task_runner_->PostTask(
 218       FROM_HERE, base::Bind(&PoolImpl::CopyVideoFrameToGpuMemoryBuffers, this,
 219                             video_frame, frame_resources, frame_ready_cb));
 220 }
 221
 222 void GpuMemoryBufferVideoFramePool::PoolImpl::OnCopiesDone(
 223     const scoped_refptr<VideoFrame>& video_frame,
 224     FrameResources* frame_resources,
 225     const FrameReadyCB& frame_ready_cb) {
 226   const VideoPixelFormat format = video_frame->format();
 227   const size_t planes = VideoFrame::NumPlanes(format);
 228   for (size_t i = 0; i < planes; ++i) {
 229     frame_resources->plane_resources[i].gpu_memory_buffer->Unmap();
 230   }
 231
 232   media_task_runner_->PostTask(
 233       FROM_HERE,
 234       base::Bind(&PoolImpl::BindAndCreateMailboxesHardwareFrameResources, this,
 235                  video_frame, frame_resources, frame_ready_cb));
 236 }
 237
 238 // Copies |video_frame| into |frame_resources| asynchronously, posting n tasks
 239 // that will be synchronized by a barrier.
 240 // After the barrier is passed OnCopiesDone will be called.
 241 void GpuMemoryBufferVideoFramePool::PoolImpl::CopyVideoFrameToGpuMemoryBuffers(
 242     const scoped_refptr<VideoFrame>& video_frame,
 243     FrameResources* frame_resources,
 244     const FrameReadyCB& frame_ready_cb) {
 245   const VideoPixelFormat format = video_frame->format();
 246   const size_t planes = VideoFrame::NumPlanes(format);
 247   gfx::Size size = video_frame->visible_rect().size();
 248   size_t copies = 0;
 249   for (size_t i = 0; i < planes; ++i) {
 250     int rows = VideoFrame::Rows(i, format, size.height());
 251     int bytes_per_row = VideoFrame::RowBytes(i, format, size.width());
 252     int rows_per_copy =
 253         std::max<size_t>(kBytesPerCopyTarget / bytes_per_row, 1);
 254     copies += rows / rows_per_copy;
 255     if (rows % rows_per_copy)
 256       ++copies;
 257   }
 258
 259   base::Closure copies_done =
 260       base::Bind(&PoolImpl::OnCopiesDone, this, video_frame, frame_resources,
 261                  frame_ready_cb);
 262   base::Closure barrier = base::BarrierClosure(copies, copies_done);
 263
 264   for (size_t i = 0; i < planes; ++i) {
 265     int rows = VideoFrame::Rows(i, format, size.height());
 266     int bytes_per_row = VideoFrame::RowBytes(i, format, size.width());
 267     int rows_per_copy =
 268         std::max<size_t>(kBytesPerCopyTarget / bytes_per_row, 1);
 269
 270     PlaneResource& plane_resource = frame_resources->plane_resources[i];
 271     void* data = nullptr;
 272     CHECK(plane_resource.gpu_memory_buffer->Map(&data));
 273     uint8* mapped_buffer = static_cast<uint8*>(data);
 274     int dest_stride = 0;
 275     plane_resource.gpu_memory_buffer->GetStride(&dest_stride);
 276
 277     for (int row = 0; row < rows; row += rows_per_copy) {
 278       worker_task_runner_->PostTask(
 279           FROM_HERE,
 280           base::Bind(&CopyRowsToBuffer, row,
 281                      std::min(rows_per_copy, rows - row), bytes_per_row,
 282                      video_frame->data(i), video_frame->stride(i),
 283                      mapped_buffer, dest_stride, barrier));
 284     }
 285   }
 286 }
 287
 288 void GpuMemoryBufferVideoFramePool::PoolImpl::
 289     BindAndCreateMailboxesHardwareFrameResources(
 290         const scoped_refptr<VideoFrame>& video_frame,
 291         FrameResources* frame_resources,
 292         const FrameReadyCB& frame_ready_cb) {
 293   gpu::gles2::GLES2Interface* gles2 = gpu_factories_->GetGLES2Interface();
 294   if (!gles2) {
 295     frame_ready_cb.Run(video_frame);
 296     return;
 297   }
 298
 299   const VideoPixelFormat format = video_frame->format();
 300   const size_t planes = VideoFrame::NumPlanes(format);
 301   const gfx::Size size = video_frame->visible_rect().size();
 302   gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes];
 303   // Set up the planes creating the mailboxes needed to refer to the textures.
 304   for (size_t i = 0; i < planes; ++i) {
 305     PlaneResource& plane_resource = frame_resources->plane_resources[i];
 306     // Bind the texture and create or rebind the image.
 307     gles2->BindTexture(texture_target_, plane_resource.texture_id);
 308
 309     if (plane_resource.gpu_memory_buffer && !plane_resource.image_id) {
 310       const size_t width = VideoFrame::Columns(i, format, size.width());
 311       const size_t height = VideoFrame::Rows(i, format, size.height());
 312       plane_resource.image_id = gles2->CreateImageCHROMIUM(
 313           plane_resource.gpu_memory_buffer->AsClientBuffer(), width, height,
 314           GL_R8_EXT);
 315     } else {
 316       gles2->ReleaseTexImage2DCHROMIUM(texture_target_,
 317                                        plane_resource.image_id);
 318     }
 319     gles2->BindTexImage2DCHROMIUM(texture_target_, plane_resource.image_id);
 320     mailbox_holders[i] =
 321         gpu::MailboxHolder(plane_resource.mailbox, texture_target_, 0);
 322   }
 323
 324   // Insert a sync_point, this is needed to make sure that the textures the
 325   // mailboxes refer to will be used only after all the previous commands posted
 326   // in the command buffer have been processed.
 327   unsigned sync_point = gles2->InsertSyncPointCHROMIUM();
 328   for (size_t i = 0; i < planes; ++i) {
 329     mailbox_holders[i].sync_point = sync_point;
 330   }
 331
 332   // Create the VideoFrame backed by native textures.
 333   scoped_refptr<VideoFrame> frame = VideoFrame::WrapYUV420NativeTextures(
 334       mailbox_holders[VideoFrame::kYPlane],
 335       mailbox_holders[VideoFrame::kUPlane],
 336       mailbox_holders[VideoFrame::kVPlane],
 337       base::Bind(&PoolImpl::MailboxHoldersReleased, this, frame_resources),
 338       size, video_frame->visible_rect(), video_frame->natural_size(),
 339       video_frame->timestamp());
 340   if (video_frame->metadata()->IsTrue(VideoFrameMetadata::ALLOW_OVERLAY))
 341     frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, true);
 342   frame_ready_cb.Run(frame);
 343 }
 344
 345 // Destroy all the resources posting one task per FrameResources
 346 // to the |media_task_runner_|.
 347 GpuMemoryBufferVideoFramePool::PoolImpl::~PoolImpl() {
 348   // Delete all the resources on the media thread.
 349   while (!resources_pool_.empty()) {
 350     FrameResources* frame_resources = resources_pool_.front();
 351     resources_pool_.pop_front();
 352     media_task_runner_->PostTask(
 353         FROM_HERE, base::Bind(&PoolImpl::DeleteFrameResources, gpu_factories_,
 354                               base::Owned(frame_resources)));
 355   }
 356 }
 357
 358 // Tries to find the resources in the pool or create them.
 359 // Incompatible resources will be dropped.
 360 GpuMemoryBufferVideoFramePool::PoolImpl::FrameResources*
 361 GpuMemoryBufferVideoFramePool::PoolImpl::GetOrCreateFrameResources(
 362     const gfx::Size& size,
 363     VideoPixelFormat format) {
 364   auto it = resources_pool_.begin();
 365   while (it != resources_pool_.end()) {
 366     FrameResources* frame_resources = *it;
 367     if (!frame_resources->in_use) {
 368       if (AreFrameResourcesCompatible(frame_resources, size, format)) {
 369         frame_resources->in_use = true;
 370         return frame_resources;
 371       } else {
 372         resources_pool_.erase(it++);
 373         DeleteFrameResources(gpu_factories_, frame_resources);
 374         delete frame_resources;
 375       }
 376     } else {
 377       it++;
 378     }
 379   }
 380
 381   // Create the resources.
 382   gpu::gles2::GLES2Interface* gles2 = gpu_factories_->GetGLES2Interface();
 383   if (!gles2)
 384     return nullptr;
 385   gles2->ActiveTexture(GL_TEXTURE0);
 386   size_t planes = VideoFrame::NumPlanes(format);
 387   FrameResources* frame_resources = new FrameResources(format, size);
 388   resources_pool_.push_back(frame_resources);
 389   for (size_t i = 0; i < planes; ++i) {
 390     PlaneResource& plane_resource = frame_resources->plane_resources[i];
 391     const size_t width = VideoFrame::Columns(i, format, size.width());
 392     const size_t height = VideoFrame::Rows(i, format, size.height());
 393     const gfx::Size plane_size(width, height);
 394     plane_resource.gpu_memory_buffer = gpu_factories_->AllocateGpuMemoryBuffer(
 395         plane_size, gfx::BufferFormat::R_8, gfx::BufferUsage::MAP);
 396
 397     gles2->GenTextures(1, &plane_resource.texture_id);
 398     gles2->BindTexture(texture_target_, plane_resource.texture_id);
 399     gles2->TexParameteri(texture_target_, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 400     gles2->TexParameteri(texture_target_, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 401     gles2->TexParameteri(texture_target_, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 402     gles2->TexParameteri(texture_target_, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 403     gles2->GenMailboxCHROMIUM(plane_resource.mailbox.name);
 404     gles2->ProduceTextureCHROMIUM(texture_target_, plane_resource.mailbox.name);
 405   }
 406   return frame_resources;
 407 }
 408
 409 // static
 410 void GpuMemoryBufferVideoFramePool::PoolImpl::DeleteFrameResources(
 411     const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories,
 412     FrameResources* frame_resources) {
 413   // TODO(dcastagna): As soon as the context lost is dealt with in media,
 414   // make sure that we won't execute this callback (use a weak pointer to
 415   // the old context).
 416   gpu::gles2::GLES2Interface* gles2 = gpu_factories->GetGLES2Interface();
 417   if (!gles2)
 418     return;
 419
 420   for (PlaneResource& plane_resource : frame_resources->plane_resources) {
 421     if (plane_resource.image_id)
 422       gles2->DestroyImageCHROMIUM(plane_resource.image_id);
 423     if (plane_resource.texture_id)
 424       gles2->DeleteTextures(1, &plane_resource.texture_id);
 425   }
 426 }
 427
 428 // Called when a VideoFrame is no longer references.
 429 void GpuMemoryBufferVideoFramePool::PoolImpl::MailboxHoldersReleased(
 430     FrameResources* frame_resources,
 431     uint32 sync_point) {
 432   // Return the resource on the media thread.
 433   media_task_runner_->PostTask(
 434       FROM_HERE,
 435       base::Bind(&PoolImpl::ReturnFrameResources, this, frame_resources));
 436 }
 437
 438 // Put back the resoruces in the pool.
 439 void GpuMemoryBufferVideoFramePool::PoolImpl::ReturnFrameResources(
 440     FrameResources* frame_resources) {
 441
 442   auto it = std::find(resources_pool_.begin(), resources_pool_.end(),
 443                       frame_resources);
 444   DCHECK(it != resources_pool_.end());
 445   // We want the pool to behave in a FIFO way.
 446   // This minimizes the chances of locking the buffer that might be
 447   // still needed for drawing.
 448   std::swap(*it, resources_pool_.back());
 449   frame_resources->in_use = false;
 450 }
 451
 452 GpuMemoryBufferVideoFramePool::GpuMemoryBufferVideoFramePool(
 453     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
 454     const scoped_refptr<base::TaskRunner>& worker_task_runner,
 455     const scoped_refptr<GpuVideoAcceleratorFactories>& gpu_factories)
 456     : pool_impl_(
 457           new PoolImpl(media_task_runner, worker_task_runner, gpu_factories)) {}
 458
 459 GpuMemoryBufferVideoFramePool::~GpuMemoryBufferVideoFramePool() {
 460 }
 461
 462 void GpuMemoryBufferVideoFramePool::MaybeCreateHardwareFrame(
 463     const scoped_refptr<VideoFrame>& video_frame,
 464     const FrameReadyCB& frame_ready_cb) {
 465   DCHECK(video_frame);
 466   pool_impl_->CreateHardwareFrame(video_frame, frame_ready_cb);
 467 }
 468
 469 }  // namespace media