cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  29  public:
  30   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
  31   ~SyncPointClientImpl() override {}
  32   uint32 InsertSyncPoint() override {
  33     return GLC(gl_, gl_->InsertSyncPointCHROMIUM());
  34   }
  35   void WaitSyncPoint(uint32 sync_point) override {
  36     GLC(gl_, gl_->WaitSyncPointCHROMIUM(sync_point));
  37   }
  38
  39  private:
  40   gpu::gles2::GLES2Interface* gl_;
  41 };
  42
  43 }  // namespace
  44
  45 VideoResourceUpdater::PlaneResource::PlaneResource(
  46     unsigned int resource_id,
  47     const gfx::Size& resource_size,
  48     ResourceFormat resource_format,
  49     gpu::Mailbox mailbox)
  50     : resource_id(resource_id),
  51       resource_size(resource_size),
  52       resource_format(resource_format),
  53       mailbox(mailbox),
  54       ref_count(0),
  55       frame_ptr(nullptr),
  56       plane_index(0) {
  57 }
  58
  59 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
  60     const PlaneResource& plane_resource,
  61     const media::VideoFrame* video_frame,
  62     int plane_index) {
  63   return plane_resource.frame_ptr == video_frame &&
  64          plane_resource.plane_index == plane_index &&
  65          plane_resource.timestamp == video_frame->timestamp();
  66 }
  67
  68 void VideoResourceUpdater::SetPlaneResourceUniqueId(
  69     const media::VideoFrame* video_frame,
  70     int plane_index,
  71     PlaneResource* plane_resource) {
  72   plane_resource->frame_ptr = video_frame;
  73   plane_resource->plane_index = plane_index;
  74   plane_resource->timestamp = video_frame->timestamp();
  75 }
  76
  77 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  78
  79 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  80
  81 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  82                                            ResourceProvider* resource_provider)
  83     : context_provider_(context_provider),
  84       resource_provider_(resource_provider) {
  85 }
  86
  87 VideoResourceUpdater::~VideoResourceUpdater() {
  88   for (const PlaneResource& plane_resource : all_resources_)
  89     resource_provider_->DeleteResource(plane_resource.resource_id);
  90 }
  91
  92 VideoResourceUpdater::ResourceList::iterator
  93 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
  94                                        ResourceFormat format,
  95                                        bool has_mailbox) {
  96   // TODO(danakj): Abstract out hw/sw resource create/delete from
  97   // ResourceProvider and stop using ResourceProvider in this class.
  98   const ResourceProvider::ResourceId resource_id =
  99       resource_provider_->CreateResource(
 100           plane_size, GL_CLAMP_TO_EDGE,
 101           ResourceProvider::TEXTURE_HINT_IMMUTABLE, format);
 102   if (resource_id == 0)
 103     return all_resources_.end();
 104
 105   gpu::Mailbox mailbox;
 106   if (has_mailbox) {
 107     DCHECK(context_provider_);
 108
 109     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 110
 111     GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
 112     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 113     GLC(gl, gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 114                                              mailbox.name));
 115   }
 116   all_resources_.push_front(
 117       PlaneResource(resource_id, plane_size, format, mailbox));
 118   return all_resources_.begin();
 119 }
 120
 121 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 122   DCHECK_EQ(resource_it->ref_count, 0);
 123   resource_provider_->DeleteResource(resource_it->resource_id);
 124   all_resources_.erase(resource_it);
 125 }
 126
 127 VideoFrameExternalResources VideoResourceUpdater::
 128     CreateExternalResourcesFromVideoFrame(
 129         const scoped_refptr<media::VideoFrame>& video_frame) {
 130   if (!VerifyFrame(video_frame))
 131     return VideoFrameExternalResources();
 132
 133   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
 134     return CreateForHardwarePlanes(video_frame);
 135   else
 136     return CreateForSoftwarePlanes(video_frame);
 137 }
 138
 139 bool VideoResourceUpdater::VerifyFrame(
 140     const scoped_refptr<media::VideoFrame>& video_frame) {
 141   switch (video_frame->format()) {
 142     // Acceptable inputs.
 143     case media::VideoFrame::YV12:
 144     case media::VideoFrame::I420:
 145     case media::VideoFrame::YV12A:
 146     case media::VideoFrame::YV16:
 147     case media::VideoFrame::YV12J:
 148     case media::VideoFrame::YV12HD:
 149     case media::VideoFrame::YV24:
 150     case media::VideoFrame::NATIVE_TEXTURE:
 151 #if defined(VIDEO_HOLE)
 152     case media::VideoFrame::HOLE:
 153 #endif  // defined(VIDEO_HOLE)
 154     case media::VideoFrame::ARGB:
 155       return true;
 156
 157     // Unacceptable inputs. ¯\(°_o)/¯
 158     case media::VideoFrame::UNKNOWN:
 159     case media::VideoFrame::NV12:
 160       break;
 161   }
 162   return false;
 163 }
 164
 165 // For frames that we receive in software format, determine the dimensions of
 166 // each plane in the frame.
 167 static gfx::Size SoftwarePlaneDimension(
 168     const scoped_refptr<media::VideoFrame>& input_frame,
 169     bool software_compositor,
 170     size_t plane_index) {
 171   if (!software_compositor) {
 172     return media::VideoFrame::PlaneSize(
 173         input_frame->format(), plane_index, input_frame->coded_size());
 174   }
 175   return input_frame->coded_size();
 176 }
 177
 178 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 179     const scoped_refptr<media::VideoFrame>& video_frame) {
 180   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 181   media::VideoFrame::Format input_frame_format = video_frame->format();
 182
 183 #if defined(VIDEO_HOLE)
 184   if (input_frame_format == media::VideoFrame::HOLE) {
 185     VideoFrameExternalResources external_resources;
 186     external_resources.type = VideoFrameExternalResources::HOLE;
 187     return external_resources;
 188   }
 189 #endif  // defined(VIDEO_HOLE)
 190
 191   // Only YUV software video frames are supported.
 192   if (input_frame_format != media::VideoFrame::YV12 &&
 193       input_frame_format != media::VideoFrame::I420 &&
 194       input_frame_format != media::VideoFrame::YV12A &&
 195       input_frame_format != media::VideoFrame::YV12J &&
 196       input_frame_format != media::VideoFrame::YV12HD &&
 197       input_frame_format != media::VideoFrame::YV16 &&
 198       input_frame_format != media::VideoFrame::YV24) {
 199     NOTREACHED() << input_frame_format;
 200     return VideoFrameExternalResources();
 201   }
 202
 203   bool software_compositor = context_provider_ == NULL;
 204
 205   ResourceFormat output_resource_format =
 206       resource_provider_->yuv_resource_format();
 207   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 208
 209   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 210   // conversion here. That involves an extra copy of each frame to a bitmap.
 211   // Obviously, this is suboptimal and should be addressed once ubercompositor
 212   // starts shaping up.
 213   if (software_compositor) {
 214     output_resource_format = kRGBResourceFormat;
 215     output_plane_count = 1;
 216   }
 217
 218   // Drop recycled resources that are the wrong format.
 219   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 220     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 221       DeleteResource(it++);
 222     else
 223       ++it;
 224   }
 225
 226   const int max_resource_size = resource_provider_->max_texture_size();
 227   std::vector<ResourceList::iterator> plane_resources;
 228   for (size_t i = 0; i < output_plane_count; ++i) {
 229     gfx::Size output_plane_resource_size =
 230         SoftwarePlaneDimension(video_frame, software_compositor, i);
 231     if (output_plane_resource_size.IsEmpty() ||
 232         output_plane_resource_size.width() > max_resource_size ||
 233         output_plane_resource_size.height() > max_resource_size) {
 234       break;
 235     }
 236
 237     // Try recycle a previously-allocated resource.
 238     ResourceList::iterator resource_it = all_resources_.end();
 239     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 240       if (it->resource_size == output_plane_resource_size &&
 241           it->resource_format == output_resource_format) {
 242         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 243           // Bingo, we found a resource that already contains the data we are
 244           // planning to put in it. It's safe to reuse it even if
 245           // resource_provider_ holds some references to it, because those
 246           // references are read-only.
 247           resource_it = it;
 248           break;
 249         }
 250
 251         // This extra check is needed because resources backed by SharedMemory
 252         // are not ref-counted, unlike mailboxes. Full discussion in
 253         // codereview.chromium.org/145273021.
 254         const bool in_use =
 255             software_compositor &&
 256             resource_provider_->InUseByConsumer(it->resource_id);
 257         if (it->ref_count == 0 && !in_use) {
 258           // We found a resource with the correct size that we can overwrite.
 259           resource_it = it;
 260         }
 261       }
 262     }
 263
 264     // Check if we need to allocate a new resource.
 265     if (resource_it == all_resources_.end()) {
 266       resource_it =
 267           AllocateResource(output_plane_resource_size, output_resource_format,
 268                            !software_compositor);
 269     }
 270     if (resource_it == all_resources_.end())
 271       break;
 272
 273     ++resource_it->ref_count;
 274     plane_resources.push_back(resource_it);
 275   }
 276
 277   if (plane_resources.size() != output_plane_count) {
 278     // Allocation failed, nothing will be returned so restore reference counts.
 279     for (ResourceList::iterator resource_it : plane_resources)
 280       --resource_it->ref_count;
 281     return VideoFrameExternalResources();
 282   }
 283
 284   VideoFrameExternalResources external_resources;
 285
 286   if (software_compositor) {
 287     DCHECK_EQ(plane_resources.size(), 1u);
 288     PlaneResource& plane_resource = *plane_resources[0];
 289     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 290     DCHECK(plane_resource.mailbox.IsZero());
 291
 292     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 293       // We need to transfer data from |video_frame| to the plane resource.
 294       if (!video_renderer_)
 295         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 296
 297       ResourceProvider::ScopedWriteLockSoftware lock(
 298           resource_provider_, plane_resource.resource_id);
 299       SkCanvas canvas(lock.sk_bitmap());
 300       // This is software path, so canvas and video_frame are always backed
 301       // by software.
 302       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 303       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 304     }
 305
 306     external_resources.software_resources.push_back(plane_resource.resource_id);
 307     external_resources.software_release_callback =
 308         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 309     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 310     return external_resources;
 311   }
 312
 313   for (size_t i = 0; i < plane_resources.size(); ++i) {
 314     PlaneResource& plane_resource = *plane_resources[i];
 315     // Update each plane's resource id with its content.
 316     DCHECK_EQ(plane_resource.resource_format,
 317               resource_provider_->yuv_resource_format());
 318
 319     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 320       // We need to transfer data from |video_frame| to the plane resource.
 321       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 322
 323       // The |resource_size_pixels| is the size of the resource we want to
 324       // upload to.
 325       gfx::Size resource_size_pixels = plane_resource.resource_size;
 326       // The |video_stride_pixels| is the width of the video frame we are
 327       // uploading (including non-frame data to fill in the stride).
 328       size_t video_stride_pixels = video_frame->stride(i);
 329
 330       size_t bytes_per_pixel = BitsPerPixel(plane_resource.resource_format) / 8;
 331       // Use 4-byte row alignment (OpenGL default) for upload performance.
 332       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 333       size_t upload_image_stride =
 334           RoundUp<size_t>(bytes_per_pixel * resource_size_pixels.width(), 4u);
 335
 336       const uint8_t* pixels;
 337       if (upload_image_stride == video_stride_pixels * bytes_per_pixel) {
 338         pixels = video_frame->data(i);
 339       } else {
 340         // Avoid malloc for each frame/plane if possible.
 341         size_t needed_size =
 342             upload_image_stride * resource_size_pixels.height();
 343         if (upload_pixels_.size() < needed_size)
 344           upload_pixels_.resize(needed_size);
 345         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 346           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 347           const uint8_t* src = video_frame->data(i) +
 348                                bytes_per_pixel * video_stride_pixels * row;
 349           memcpy(dst, src, resource_size_pixels.width() * bytes_per_pixel);
 350         }
 351         pixels = &upload_pixels_[0];
 352       }
 353
 354       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 355                                          resource_size_pixels);
 356       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 357     }
 358
 359     external_resources.mailboxes.push_back(
 360         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 361     external_resources.release_callbacks.push_back(
 362         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 363   }
 364
 365   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 366   return external_resources;
 367 }
 368
 369 // static
 370 void VideoResourceUpdater::ReturnTexture(
 371     base::WeakPtr<VideoResourceUpdater> updater,
 372     const scoped_refptr<media::VideoFrame>& video_frame,
 373     uint32 sync_point,
 374     bool lost_resource,
 375     BlockingTaskRunner* main_thread_task_runner) {
 376   // TODO(dshwang) this case should be forwarded to the decoder as lost
 377   // resource.
 378   if (lost_resource || !updater.get())
 379     return;
 380   // VideoFrame::UpdateReleaseSyncPoint() creates new sync point using the same
 381   // GL context which created the given |sync_point|, so discard the
 382   // |sync_point|.
 383   SyncPointClientImpl client(updater->context_provider_->ContextGL());
 384   video_frame->UpdateReleaseSyncPoint(&client);
 385 }
 386
 387 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 388     const scoped_refptr<media::VideoFrame>& video_frame) {
 389   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 390   media::VideoFrame::Format frame_format = video_frame->format();
 391
 392   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
 393   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
 394       return VideoFrameExternalResources();
 395
 396   if (!context_provider_)
 397     return VideoFrameExternalResources();
 398
 399   const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
 400   VideoFrameExternalResources external_resources;
 401   switch (mailbox_holder->texture_target) {
 402     case GL_TEXTURE_2D:
 403       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
 404       break;
 405     case GL_TEXTURE_EXTERNAL_OES:
 406       external_resources.type =
 407           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 408       break;
 409     case GL_TEXTURE_RECTANGLE_ARB:
 410       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 411       break;
 412     default:
 413       NOTREACHED();
 414       return VideoFrameExternalResources();
 415   }
 416
 417   external_resources.mailboxes.push_back(
 418       TextureMailbox(mailbox_holder->mailbox,
 419                      mailbox_holder->texture_target,
 420                      mailbox_holder->sync_point));
 421   external_resources.mailboxes.back().set_allow_overlay(
 422       video_frame->allow_overlay());
 423   external_resources.release_callbacks.push_back(
 424       base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 425   return external_resources;
 426 }
 427
 428 // static
 429 void VideoResourceUpdater::RecycleResource(
 430     base::WeakPtr<VideoResourceUpdater> updater,
 431     ResourceProvider::ResourceId resource_id,
 432     uint32 sync_point,
 433     bool lost_resource,
 434     BlockingTaskRunner* main_thread_task_runner) {
 435   if (!updater.get()) {
 436     // Resource was already deleted.
 437     return;
 438   }
 439
 440   const ResourceList::iterator resource_it = std::find_if(
 441       updater->all_resources_.begin(), updater->all_resources_.end(),
 442       [resource_id](const PlaneResource& plane_resource) {
 443         return plane_resource.resource_id == resource_id;
 444       });
 445   if (resource_it == updater->all_resources_.end())
 446     return;
 447
 448   ContextProvider* context_provider = updater->context_provider_;
 449   if (context_provider && sync_point) {
 450     GLC(context_provider->ContextGL(),
 451         context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
 452   }
 453
 454   if (lost_resource) {
 455     resource_it->ref_count = 0;
 456     updater->DeleteResource(resource_it);
 457     return;
 458   }
 459
 460   --resource_it->ref_count;
 461   DCHECK_GE(resource_it->ref_count, 0);
 462 }
 463
 464 }  // namespace cc