cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/math_util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  29  public:
  30   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl,
  31                                uint32 sync_point)
  32       : gl_(gl), sync_point_(sync_point) {}
  33   ~SyncPointClientImpl() override {}
  34   uint32 InsertSyncPoint() override {
  35     if (sync_point_)
  36       return sync_point_;
  37     return gl_->InsertSyncPointCHROMIUM();
  38   }
  39   void WaitSyncPoint(uint32 sync_point) override {
  40     if (!sync_point)
  41       return;
  42     gl_->WaitSyncPointCHROMIUM(sync_point);
  43     if (sync_point_) {
  44       gl_->WaitSyncPointCHROMIUM(sync_point_);
  45       sync_point_ = 0;
  46     }
  47   }
  48
  49  private:
  50   gpu::gles2::GLES2Interface* gl_;
  51   uint32 sync_point_;
  52 };
  53
  54 }  // namespace
  55
  56 VideoResourceUpdater::PlaneResource::PlaneResource(
  57     unsigned int resource_id,
  58     const gfx::Size& resource_size,
  59     ResourceFormat resource_format,
  60     gpu::Mailbox mailbox)
  61     : resource_id(resource_id),
  62       resource_size(resource_size),
  63       resource_format(resource_format),
  64       mailbox(mailbox),
  65       ref_count(0),
  66       frame_ptr(nullptr),
  67       plane_index(0u) {
  68 }
  69
  70 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
  71     const PlaneResource& plane_resource,
  72     const media::VideoFrame* video_frame,
  73     size_t plane_index) {
  74   return plane_resource.frame_ptr == video_frame &&
  75          plane_resource.plane_index == plane_index &&
  76          plane_resource.timestamp == video_frame->timestamp();
  77 }
  78
  79 void VideoResourceUpdater::SetPlaneResourceUniqueId(
  80     const media::VideoFrame* video_frame,
  81     size_t plane_index,
  82     PlaneResource* plane_resource) {
  83   plane_resource->frame_ptr = video_frame;
  84   plane_resource->plane_index = plane_index;
  85   plane_resource->timestamp = video_frame->timestamp();
  86 }
  87
  88 VideoFrameExternalResources::VideoFrameExternalResources()
  89     : type(NONE), read_lock_fences_enabled(false) {
  90 }
  91
  92 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  93
  94 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  95                                            ResourceProvider* resource_provider)
  96     : context_provider_(context_provider),
  97       resource_provider_(resource_provider) {
  98 }
  99
 100 VideoResourceUpdater::~VideoResourceUpdater() {
 101   for (const PlaneResource& plane_resource : all_resources_)
 102     resource_provider_->DeleteResource(plane_resource.resource_id);
 103 }
 104
 105 VideoResourceUpdater::ResourceList::iterator
 106 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
 107                                        ResourceFormat format,
 108                                        bool has_mailbox) {
 109   // TODO(danakj): Abstract out hw/sw resource create/delete from
 110   // ResourceProvider and stop using ResourceProvider in this class.
 111   const ResourceId resource_id = resource_provider_->CreateResource(
 112       plane_size, GL_CLAMP_TO_EDGE, ResourceProvider::TEXTURE_HINT_IMMUTABLE,
 113       format);
 114   if (resource_id == 0)
 115     return all_resources_.end();
 116
 117   gpu::Mailbox mailbox;
 118   if (has_mailbox) {
 119     DCHECK(context_provider_);
 120
 121     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 122
 123     gl->GenMailboxCHROMIUM(mailbox.name);
 124     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 125     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 126                                      mailbox.name);
 127   }
 128   all_resources_.push_front(
 129       PlaneResource(resource_id, plane_size, format, mailbox));
 130   return all_resources_.begin();
 131 }
 132
 133 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 134   DCHECK_EQ(resource_it->ref_count, 0);
 135   resource_provider_->DeleteResource(resource_it->resource_id);
 136   all_resources_.erase(resource_it);
 137 }
 138
 139 VideoFrameExternalResources VideoResourceUpdater::
 140     CreateExternalResourcesFromVideoFrame(
 141         const scoped_refptr<media::VideoFrame>& video_frame) {
 142   if (video_frame->format() == media::PIXEL_FORMAT_UNKNOWN)
 143     return VideoFrameExternalResources();
 144   DCHECK(video_frame->HasTextures() || video_frame->IsMappable());
 145   if (video_frame->HasTextures())
 146     return CreateForHardwarePlanes(video_frame);
 147   else
 148     return CreateForSoftwarePlanes(video_frame);
 149 }
 150
 151 // For frames that we receive in software format, determine the dimensions of
 152 // each plane in the frame.
 153 static gfx::Size SoftwarePlaneDimension(
 154     const scoped_refptr<media::VideoFrame>& input_frame,
 155     bool software_compositor,
 156     size_t plane_index) {
 157   if (!software_compositor) {
 158     return media::VideoFrame::PlaneSize(
 159         input_frame->format(), plane_index, input_frame->coded_size());
 160   }
 161   return input_frame->coded_size();
 162 }
 163
 164 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 165     const scoped_refptr<media::VideoFrame>& video_frame) {
 166   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 167   const media::VideoPixelFormat input_frame_format = video_frame->format();
 168
 169 #if defined(VIDEO_HOLE)
 170   if (video_frame->storage_type() == media::VideoFrame::STORAGE_HOLE) {
 171     VideoFrameExternalResources external_resources;
 172     external_resources.type = VideoFrameExternalResources::HOLE;
 173     return external_resources;
 174   }
 175 #endif  // defined(VIDEO_HOLE)
 176
 177   // Only YUV software video frames are supported.
 178   if (!media::IsYuvPlanar(input_frame_format)) {
 179     NOTREACHED() << media::VideoPixelFormatToString(input_frame_format);
 180     return VideoFrameExternalResources();
 181   }
 182
 183   const bool software_compositor = context_provider_ == NULL;
 184
 185   ResourceFormat output_resource_format =
 186       resource_provider_->yuv_resource_format();
 187   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 188
 189   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 190   // conversion here. That involves an extra copy of each frame to a bitmap.
 191   // Obviously, this is suboptimal and should be addressed once ubercompositor
 192   // starts shaping up.
 193   if (software_compositor) {
 194     output_resource_format = kRGBResourceFormat;
 195     output_plane_count = 1;
 196   }
 197
 198   // Drop recycled resources that are the wrong format.
 199   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 200     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 201       DeleteResource(it++);
 202     else
 203       ++it;
 204   }
 205
 206   const int max_resource_size = resource_provider_->max_texture_size();
 207   std::vector<ResourceList::iterator> plane_resources;
 208   for (size_t i = 0; i < output_plane_count; ++i) {
 209     gfx::Size output_plane_resource_size =
 210         SoftwarePlaneDimension(video_frame, software_compositor, i);
 211     if (output_plane_resource_size.IsEmpty() ||
 212         output_plane_resource_size.width() > max_resource_size ||
 213         output_plane_resource_size.height() > max_resource_size) {
 214       break;
 215     }
 216
 217     // Try recycle a previously-allocated resource.
 218     ResourceList::iterator resource_it = all_resources_.end();
 219     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 220       if (it->resource_size == output_plane_resource_size &&
 221           it->resource_format == output_resource_format) {
 222         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 223           // Bingo, we found a resource that already contains the data we are
 224           // planning to put in it. It's safe to reuse it even if
 225           // resource_provider_ holds some references to it, because those
 226           // references are read-only.
 227           resource_it = it;
 228           break;
 229         }
 230
 231         // This extra check is needed because resources backed by SharedMemory
 232         // are not ref-counted, unlike mailboxes. Full discussion in
 233         // codereview.chromium.org/145273021.
 234         const bool in_use =
 235             software_compositor &&
 236             resource_provider_->InUseByConsumer(it->resource_id);
 237         if (it->ref_count == 0 && !in_use) {
 238           // We found a resource with the correct size that we can overwrite.
 239           resource_it = it;
 240         }
 241       }
 242     }
 243
 244     // Check if we need to allocate a new resource.
 245     if (resource_it == all_resources_.end()) {
 246       resource_it =
 247           AllocateResource(output_plane_resource_size, output_resource_format,
 248                            !software_compositor);
 249     }
 250     if (resource_it == all_resources_.end())
 251       break;
 252
 253     ++resource_it->ref_count;
 254     plane_resources.push_back(resource_it);
 255   }
 256
 257   if (plane_resources.size() != output_plane_count) {
 258     // Allocation failed, nothing will be returned so restore reference counts.
 259     for (ResourceList::iterator resource_it : plane_resources)
 260       --resource_it->ref_count;
 261     return VideoFrameExternalResources();
 262   }
 263
 264   VideoFrameExternalResources external_resources;
 265
 266   if (software_compositor) {
 267     DCHECK_EQ(plane_resources.size(), 1u);
 268     PlaneResource& plane_resource = *plane_resources[0];
 269     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 270     DCHECK(plane_resource.mailbox.IsZero());
 271
 272     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 273       // We need to transfer data from |video_frame| to the plane resource.
 274       if (!video_renderer_)
 275         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 276
 277       ResourceProvider::ScopedWriteLockSoftware lock(
 278           resource_provider_, plane_resource.resource_id);
 279       SkCanvas canvas(lock.sk_bitmap());
 280       // This is software path, so canvas and video_frame are always backed
 281       // by software.
 282       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 283       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 284     }
 285
 286     external_resources.software_resources.push_back(plane_resource.resource_id);
 287     external_resources.software_release_callback =
 288         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 289     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 290     return external_resources;
 291   }
 292
 293   for (size_t i = 0; i < plane_resources.size(); ++i) {
 294     PlaneResource& plane_resource = *plane_resources[i];
 295     // Update each plane's resource id with its content.
 296     DCHECK_EQ(plane_resource.resource_format,
 297               resource_provider_->yuv_resource_format());
 298
 299     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 300       // We need to transfer data from |video_frame| to the plane resource.
 301       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 302
 303       // The |resource_size_pixels| is the size of the resource we want to
 304       // upload to.
 305       gfx::Size resource_size_pixels = plane_resource.resource_size;
 306       // The |video_stride_pixels| is the width of the video frame we are
 307       // uploading (including non-frame data to fill in the stride).
 308       int video_stride_pixels = video_frame->stride(i);
 309
 310       size_t bytes_per_row = ResourceUtil::UncheckedWidthInBytes<size_t>(
 311           resource_size_pixels.width(), plane_resource.resource_format);
 312       // Use 4-byte row alignment (OpenGL default) for upload performance.
 313       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 314       size_t upload_image_stride =
 315           MathUtil::UncheckedRoundUp<size_t>(bytes_per_row, 4u);
 316
 317       const uint8_t* pixels;
 318       size_t video_bytes_per_row = ResourceUtil::UncheckedWidthInBytes<size_t>(
 319           video_stride_pixels, plane_resource.resource_format);
 320       if (upload_image_stride == video_bytes_per_row) {
 321         pixels = video_frame->data(i);
 322       } else {
 323         // Avoid malloc for each frame/plane if possible.
 324         size_t needed_size =
 325             upload_image_stride * resource_size_pixels.height();
 326         if (upload_pixels_.size() < needed_size)
 327           upload_pixels_.resize(needed_size);
 328         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 329           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 330           const uint8_t* src =
 331               video_frame->data(i) + (video_bytes_per_row * row);
 332           memcpy(dst, src, bytes_per_row);
 333         }
 334         pixels = &upload_pixels_[0];
 335       }
 336
 337       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 338                                          resource_size_pixels);
 339       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 340     }
 341
 342     external_resources.mailboxes.push_back(
 343         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 344     external_resources.release_callbacks.push_back(
 345         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 346   }
 347
 348   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 349   return external_resources;
 350 }
 351
 352 // static
 353 void VideoResourceUpdater::ReturnTexture(
 354     base::WeakPtr<VideoResourceUpdater> updater,
 355     const scoped_refptr<media::VideoFrame>& video_frame,
 356     uint32 sync_point,
 357     bool lost_resource,
 358     BlockingTaskRunner* main_thread_task_runner) {
 359   // TODO(dshwang) this case should be forwarded to the decoder as lost
 360   // resource.
 361   if (lost_resource || !updater.get())
 362     return;
 363   // Update the release sync point in |video_frame| with |sync_point|
 364   // returned by the compositor and emit a WaitSyncPointCHROMIUM on
 365   // |video_frame|'s previous sync point using the current GL context.
 366   SyncPointClientImpl client(updater->context_provider_->ContextGL(),
 367                              sync_point);
 368   video_frame->UpdateReleaseSyncPoint(&client);
 369 }
 370
 371 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 372     const scoped_refptr<media::VideoFrame>& video_frame) {
 373   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 374   DCHECK(video_frame->HasTextures());
 375   if (!context_provider_)
 376     return VideoFrameExternalResources();
 377
 378   const size_t textures = media::VideoFrame::NumPlanes(video_frame->format());
 379   DCHECK_GE(textures, 1u);
 380   VideoFrameExternalResources external_resources;
 381   external_resources.read_lock_fences_enabled = true;
 382   switch (video_frame->format()) {
 383     case media::PIXEL_FORMAT_ARGB:
 384     case media::PIXEL_FORMAT_XRGB:
 385       DCHECK_EQ(1u, textures);
 386       switch (video_frame->mailbox_holder(0).texture_target) {
 387         case GL_TEXTURE_2D:
 388           external_resources.type =
 389               (video_frame->format() == media::PIXEL_FORMAT_XRGB)
 390                   ? VideoFrameExternalResources::RGB_RESOURCE
 391                   : VideoFrameExternalResources::RGBA_RESOURCE;
 392           break;
 393         case GL_TEXTURE_EXTERNAL_OES:
 394           external_resources.type =
 395               VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 396           break;
 397         case GL_TEXTURE_RECTANGLE_ARB:
 398           external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 399           break;
 400         default:
 401           NOTREACHED();
 402           return VideoFrameExternalResources();
 403       }
 404       break;
 405     case media::PIXEL_FORMAT_I420:
 406       external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 407       break;
 408 #if defined(OS_MACOSX) || defined(OS_CHROMEOS)
 409     case media::PIXEL_FORMAT_NV12:
 410 #endif
 411     case media::PIXEL_FORMAT_YV12:
 412     case media::PIXEL_FORMAT_YV16:
 413     case media::PIXEL_FORMAT_YV24:
 414     case media::PIXEL_FORMAT_YV12A:
 415     case media::PIXEL_FORMAT_UNKNOWN:
 416       DLOG(ERROR) << "Unsupported Texture format"
 417                   << media::VideoPixelFormatToString(video_frame->format());
 418       return external_resources;
 419   }
 420   DCHECK_NE(VideoFrameExternalResources::NONE, external_resources.type);
 421
 422   for (size_t i = 0; i < textures; ++i) {
 423     const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
 424     external_resources.mailboxes.push_back(
 425         TextureMailbox(mailbox_holder.mailbox, mailbox_holder.texture_target,
 426                        mailbox_holder.sync_point, video_frame->coded_size(),
 427                        video_frame->metadata()->IsTrue(
 428                            media::VideoFrameMetadata::ALLOW_OVERLAY)));
 429     external_resources.release_callbacks.push_back(
 430         base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 431   }
 432   return external_resources;
 433 }
 434
 435 // static
 436 void VideoResourceUpdater::RecycleResource(
 437     base::WeakPtr<VideoResourceUpdater> updater,
 438     ResourceId resource_id,
 439     uint32 sync_point,
 440     bool lost_resource,
 441     BlockingTaskRunner* main_thread_task_runner) {
 442   if (!updater.get()) {
 443     // Resource was already deleted.
 444     return;
 445   }
 446
 447   const ResourceList::iterator resource_it = std::find_if(
 448       updater->all_resources_.begin(), updater->all_resources_.end(),
 449       [resource_id](const PlaneResource& plane_resource) {
 450         return plane_resource.resource_id == resource_id;
 451       });
 452   if (resource_it == updater->all_resources_.end())
 453     return;
 454
 455   ContextProvider* context_provider = updater->context_provider_;
 456   if (context_provider && sync_point) {
 457     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 458   }
 459
 460   if (lost_resource) {
 461     resource_it->ref_count = 0;
 462     updater->DeleteResource(resource_it);
 463     return;
 464   }
 465
 466   --resource_it->ref_count;
 467   DCHECK_GE(resource_it->ref_count, 0);
 468 }
 469
 470 }  // namespace cc