cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/math_util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  29  public:
  30   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl,
  31                                uint32 sync_point)
  32       : gl_(gl), sync_point_(sync_point) {}
  33   ~SyncPointClientImpl() override {}
  34   uint32 InsertSyncPoint() override {
  35     if (sync_point_)
  36       return sync_point_;
  37     return gl_->InsertSyncPointCHROMIUM();
  38   }
  39   void WaitSyncPoint(uint32 sync_point) override {
  40     if (!sync_point)
  41       return;
  42     gl_->WaitSyncPointCHROMIUM(sync_point);
  43     if (sync_point_) {
  44       gl_->WaitSyncPointCHROMIUM(sync_point_);
  45       sync_point_ = 0;
  46     }
  47   }
  48
  49  private:
  50   gpu::gles2::GLES2Interface* gl_;
  51   uint32 sync_point_;
  52 };
  53
  54 }  // namespace
  55
  56 VideoResourceUpdater::PlaneResource::PlaneResource(
  57     unsigned int resource_id,
  58     const gfx::Size& resource_size,
  59     ResourceFormat resource_format,
  60     gpu::Mailbox mailbox)
  61     : resource_id(resource_id),
  62       resource_size(resource_size),
  63       resource_format(resource_format),
  64       mailbox(mailbox),
  65       ref_count(0),
  66       frame_ptr(nullptr),
  67       plane_index(0u) {
  68 }
  69
  70 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
  71     const PlaneResource& plane_resource,
  72     const media::VideoFrame* video_frame,
  73     size_t plane_index) {
  74   return plane_resource.frame_ptr == video_frame &&
  75          plane_resource.plane_index == plane_index &&
  76          plane_resource.timestamp == video_frame->timestamp();
  77 }
  78
  79 void VideoResourceUpdater::SetPlaneResourceUniqueId(
  80     const media::VideoFrame* video_frame,
  81     size_t plane_index,
  82     PlaneResource* plane_resource) {
  83   plane_resource->frame_ptr = video_frame;
  84   plane_resource->plane_index = plane_index;
  85   plane_resource->timestamp = video_frame->timestamp();
  86 }
  87
  88 VideoFrameExternalResources::VideoFrameExternalResources()
  89     : type(NONE), read_lock_fences_enabled(false) {
  90 }
  91
  92 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  93
  94 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  95                                            ResourceProvider* resource_provider)
  96     : context_provider_(context_provider),
  97       resource_provider_(resource_provider) {
  98 }
  99
 100 VideoResourceUpdater::~VideoResourceUpdater() {
 101   for (const PlaneResource& plane_resource : all_resources_)
 102     resource_provider_->DeleteResource(plane_resource.resource_id);
 103 }
 104
 105 VideoResourceUpdater::ResourceList::iterator
 106 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
 107                                        ResourceFormat format,
 108                                        bool has_mailbox) {
 109   // TODO(danakj): Abstract out hw/sw resource create/delete from
 110   // ResourceProvider and stop using ResourceProvider in this class.
 111   const ResourceId resource_id = resource_provider_->CreateResource(
 112       plane_size, GL_CLAMP_TO_EDGE, ResourceProvider::TEXTURE_HINT_IMMUTABLE,
 113       format);
 114   if (resource_id == 0)
 115     return all_resources_.end();
 116
 117   gpu::Mailbox mailbox;
 118   if (has_mailbox) {
 119     DCHECK(context_provider_);
 120
 121     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 122
 123     gl->GenMailboxCHROMIUM(mailbox.name);
 124     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 125     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 126                                      mailbox.name);
 127   }
 128   all_resources_.push_front(
 129       PlaneResource(resource_id, plane_size, format, mailbox));
 130   return all_resources_.begin();
 131 }
 132
 133 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 134   DCHECK_EQ(resource_it->ref_count, 0);
 135   resource_provider_->DeleteResource(resource_it->resource_id);
 136   all_resources_.erase(resource_it);
 137 }
 138
 139 VideoFrameExternalResources VideoResourceUpdater::
 140     CreateExternalResourcesFromVideoFrame(
 141         const scoped_refptr<media::VideoFrame>& video_frame) {
 142   if (video_frame->format() == media::PIXEL_FORMAT_UNKNOWN)
 143     return VideoFrameExternalResources();
 144   DCHECK(video_frame->HasTextures() || video_frame->IsMappable());
 145   if (video_frame->HasTextures())
 146     return CreateForHardwarePlanes(video_frame);
 147   else
 148     return CreateForSoftwarePlanes(video_frame);
 149 }
 150
 151 // For frames that we receive in software format, determine the dimensions of
 152 // each plane in the frame.
 153 static gfx::Size SoftwarePlaneDimension(
 154     const scoped_refptr<media::VideoFrame>& input_frame,
 155     bool software_compositor,
 156     size_t plane_index) {
 157   if (!software_compositor) {
 158     return media::VideoFrame::PlaneSize(
 159         input_frame->format(), plane_index, input_frame->coded_size());
 160   }
 161   return input_frame->coded_size();
 162 }
 163
 164 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 165     const scoped_refptr<media::VideoFrame>& video_frame) {
 166   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 167   const media::VideoPixelFormat input_frame_format = video_frame->format();
 168
 169 #if defined(VIDEO_HOLE)
 170   if (video_frame->storage_type() == media::VideoFrame::STORAGE_HOLE) {
 171     VideoFrameExternalResources external_resources;
 172     external_resources.type = VideoFrameExternalResources::HOLE;
 173     return external_resources;
 174   }
 175 #endif  // defined(VIDEO_HOLE)
 176
 177   // Only YUV software video frames are supported.
 178   if (!media::IsYuvPlanar(input_frame_format)) {
 179     NOTREACHED() << media::VideoPixelFormatToString(input_frame_format);
 180     return VideoFrameExternalResources();
 181   }
 182
 183   const bool software_compositor = context_provider_ == NULL;
 184
 185   ResourceFormat output_resource_format =
 186       resource_provider_->yuv_resource_format();
 187   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 188
 189   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 190   // conversion here. That involves an extra copy of each frame to a bitmap.
 191   // Obviously, this is suboptimal and should be addressed once ubercompositor
 192   // starts shaping up.
 193   if (software_compositor) {
 194     output_resource_format = kRGBResourceFormat;
 195     output_plane_count = 1;
 196   }
 197
 198   // Drop recycled resources that are the wrong format.
 199   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 200     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 201       DeleteResource(it++);
 202     else
 203       ++it;
 204   }
 205
 206   const int max_resource_size = resource_provider_->max_texture_size();
 207   std::vector<ResourceList::iterator> plane_resources;
 208   for (size_t i = 0; i < output_plane_count; ++i) {
 209     gfx::Size output_plane_resource_size =
 210         SoftwarePlaneDimension(video_frame, software_compositor, i);
 211     if (output_plane_resource_size.IsEmpty() ||
 212         output_plane_resource_size.width() > max_resource_size ||
 213         output_plane_resource_size.height() > max_resource_size) {
 214       break;
 215     }
 216
 217     // Try recycle a previously-allocated resource.
 218     ResourceList::iterator resource_it = all_resources_.end();
 219     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 220       if (it->resource_size == output_plane_resource_size &&
 221           it->resource_format == output_resource_format) {
 222         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 223           // Bingo, we found a resource that already contains the data we are
 224           // planning to put in it. It's safe to reuse it even if
 225           // resource_provider_ holds some references to it, because those
 226           // references are read-only.
 227           resource_it = it;
 228           break;
 229         }
 230
 231         // This extra check is needed because resources backed by SharedMemory
 232         // are not ref-counted, unlike mailboxes. Full discussion in
 233         // codereview.chromium.org/145273021.
 234         const bool in_use =
 235             software_compositor &&
 236             resource_provider_->InUseByConsumer(it->resource_id);
 237         if (it->ref_count == 0 && !in_use) {
 238           // We found a resource with the correct size that we can overwrite.
 239           resource_it = it;
 240         }
 241       }
 242     }
 243
 244     // Check if we need to allocate a new resource.
 245     if (resource_it == all_resources_.end()) {
 246       resource_it =
 247           AllocateResource(output_plane_resource_size, output_resource_format,
 248                            !software_compositor);
 249     }
 250     if (resource_it == all_resources_.end())
 251       break;
 252
 253     ++resource_it->ref_count;
 254     plane_resources.push_back(resource_it);
 255   }
 256
 257   if (plane_resources.size() != output_plane_count) {
 258     // Allocation failed, nothing will be returned so restore reference counts.
 259     for (ResourceList::iterator resource_it : plane_resources)
 260       --resource_it->ref_count;
 261     return VideoFrameExternalResources();
 262   }
 263
 264   VideoFrameExternalResources external_resources;
 265
 266   if (software_compositor) {
 267     DCHECK_EQ(plane_resources.size(), 1u);
 268     PlaneResource& plane_resource = *plane_resources[0];
 269     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 270     DCHECK(plane_resource.mailbox.IsZero());
 271
 272     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 273       // We need to transfer data from |video_frame| to the plane resource.
 274       if (!video_renderer_)
 275         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 276
 277       ResourceProvider::ScopedWriteLockSoftware lock(
 278           resource_provider_, plane_resource.resource_id);
 279       SkCanvas canvas(lock.sk_bitmap());
 280       // This is software path, so canvas and video_frame are always backed
 281       // by software.
 282       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 283       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 284     }
 285
 286     external_resources.software_resources.push_back(plane_resource.resource_id);
 287     external_resources.software_release_callback =
 288         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 289     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 290     return external_resources;
 291   }
 292
 293   for (size_t i = 0; i < plane_resources.size(); ++i) {
 294     PlaneResource& plane_resource = *plane_resources[i];
 295     // Update each plane's resource id with its content.
 296     DCHECK_EQ(plane_resource.resource_format,
 297               resource_provider_->yuv_resource_format());
 298
 299     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 300       // We need to transfer data from |video_frame| to the plane resource.
 301       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 302
 303       // The |resource_size_pixels| is the size of the resource we want to
 304       // upload to.
 305       gfx::Size resource_size_pixels = plane_resource.resource_size;
 306       // The |video_stride_pixels| is the width of the video frame we are
 307       // uploading (including non-frame data to fill in the stride).
 308       size_t video_stride_pixels = video_frame->stride(i);
 309
 310       size_t bytes_per_pixel = BitsPerPixel(plane_resource.resource_format) / 8;
 311       // Use 4-byte row alignment (OpenGL default) for upload performance.
 312       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 313       size_t upload_image_stride = MathUtil::RoundUp<size_t>(
 314           bytes_per_pixel * resource_size_pixels.width(), 4u);
 315
 316       const uint8_t* pixels;
 317       if (upload_image_stride == video_stride_pixels * bytes_per_pixel) {
 318         pixels = video_frame->data(i);
 319       } else {
 320         // Avoid malloc for each frame/plane if possible.
 321         size_t needed_size =
 322             upload_image_stride * resource_size_pixels.height();
 323         if (upload_pixels_.size() < needed_size)
 324           upload_pixels_.resize(needed_size);
 325         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 326           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 327           const uint8_t* src = video_frame->data(i) +
 328                                bytes_per_pixel * video_stride_pixels * row;
 329           memcpy(dst, src, resource_size_pixels.width() * bytes_per_pixel);
 330         }
 331         pixels = &upload_pixels_[0];
 332       }
 333
 334       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 335                                          resource_size_pixels);
 336       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 337     }
 338
 339     external_resources.mailboxes.push_back(
 340         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 341     external_resources.release_callbacks.push_back(
 342         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 343   }
 344
 345   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 346   return external_resources;
 347 }
 348
 349 // static
 350 void VideoResourceUpdater::ReturnTexture(
 351     base::WeakPtr<VideoResourceUpdater> updater,
 352     const scoped_refptr<media::VideoFrame>& video_frame,
 353     uint32 sync_point,
 354     bool lost_resource,
 355     BlockingTaskRunner* main_thread_task_runner) {
 356   // TODO(dshwang) this case should be forwarded to the decoder as lost
 357   // resource.
 358   if (lost_resource || !updater.get())
 359     return;
 360   // Update the release sync point in |video_frame| with |sync_point|
 361   // returned by the compositor and emit a WaitSyncPointCHROMIUM on
 362   // |video_frame|'s previous sync point using the current GL context.
 363   SyncPointClientImpl client(updater->context_provider_->ContextGL(),
 364                              sync_point);
 365   video_frame->UpdateReleaseSyncPoint(&client);
 366 }
 367
 368 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 369     const scoped_refptr<media::VideoFrame>& video_frame) {
 370   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 371   DCHECK(video_frame->HasTextures());
 372   if (!context_provider_)
 373     return VideoFrameExternalResources();
 374
 375   const size_t textures = media::VideoFrame::NumPlanes(video_frame->format());
 376   DCHECK_GE(textures, 1u);
 377   VideoFrameExternalResources external_resources;
 378   external_resources.read_lock_fences_enabled = true;
 379   switch (video_frame->format()) {
 380     case media::PIXEL_FORMAT_ARGB:
 381     case media::PIXEL_FORMAT_XRGB:
 382       DCHECK_EQ(1u, textures);
 383       switch (video_frame->mailbox_holder(0).texture_target) {
 384         case GL_TEXTURE_2D:
 385           external_resources.type =
 386               (video_frame->format() == media::PIXEL_FORMAT_XRGB)
 387                   ? VideoFrameExternalResources::RGB_RESOURCE
 388                   : VideoFrameExternalResources::RGBA_RESOURCE;
 389           break;
 390         case GL_TEXTURE_EXTERNAL_OES:
 391           external_resources.type =
 392               VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 393           break;
 394         case GL_TEXTURE_RECTANGLE_ARB:
 395           external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 396           break;
 397         default:
 398           NOTREACHED();
 399           return VideoFrameExternalResources();
 400       }
 401       break;
 402     case media::PIXEL_FORMAT_I420:
 403       external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 404       break;
 405 #if defined(OS_MACOSX) || defined(OS_CHROMEOS)
 406     case media::PIXEL_FORMAT_NV12:
 407 #endif
 408     case media::PIXEL_FORMAT_YV12:
 409     case media::PIXEL_FORMAT_YV16:
 410     case media::PIXEL_FORMAT_YV24:
 411     case media::PIXEL_FORMAT_YV12A:
 412     case media::PIXEL_FORMAT_UNKNOWN:
 413       DLOG(ERROR) << "Unsupported Texture format"
 414                   << media::VideoPixelFormatToString(video_frame->format());
 415       return external_resources;
 416   }
 417   DCHECK_NE(VideoFrameExternalResources::NONE, external_resources.type);
 418
 419   for (size_t i = 0; i < textures; ++i) {
 420     const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
 421     external_resources.mailboxes.push_back(
 422         TextureMailbox(mailbox_holder.mailbox, mailbox_holder.texture_target,
 423                        mailbox_holder.sync_point, video_frame->coded_size(),
 424                        video_frame->metadata()->IsTrue(
 425                            media::VideoFrameMetadata::ALLOW_OVERLAY)));
 426     external_resources.release_callbacks.push_back(
 427         base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 428   }
 429   return external_resources;
 430 }
 431
 432 // static
 433 void VideoResourceUpdater::RecycleResource(
 434     base::WeakPtr<VideoResourceUpdater> updater,
 435     ResourceId resource_id,
 436     uint32 sync_point,
 437     bool lost_resource,
 438     BlockingTaskRunner* main_thread_task_runner) {
 439   if (!updater.get()) {
 440     // Resource was already deleted.
 441     return;
 442   }
 443
 444   const ResourceList::iterator resource_it = std::find_if(
 445       updater->all_resources_.begin(), updater->all_resources_.end(),
 446       [resource_id](const PlaneResource& plane_resource) {
 447         return plane_resource.resource_id == resource_id;
 448       });
 449   if (resource_it == updater->all_resources_.end())
 450     return;
 451
 452   ContextProvider* context_provider = updater->context_provider_;
 453   if (context_provider && sync_point) {
 454     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 455   }
 456
 457   if (lost_resource) {
 458     resource_it->ref_count = 0;
 459     updater->DeleteResource(resource_it);
 460     return;
 461   }
 462
 463   --resource_it->ref_count;
 464   DCHECK_GE(resource_it->ref_count, 0);
 465 }
 466
 467 }  // namespace cc