cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/math_util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 VideoFrameExternalResources::ResourceType ResourceTypeForVideoFrame(
  29     media::VideoFrame* video_frame) {
  30   switch (video_frame->format()) {
  31     case media::PIXEL_FORMAT_ARGB:
  32     case media::PIXEL_FORMAT_XRGB:
  33     case media::PIXEL_FORMAT_UYVY:
  34       switch (video_frame->mailbox_holder(0).texture_target) {
  35         case GL_TEXTURE_2D:
  36           return (video_frame->format() == media::PIXEL_FORMAT_XRGB)
  37                      ? VideoFrameExternalResources::RGB_RESOURCE
  38                      : VideoFrameExternalResources::RGBA_RESOURCE;
  39         case GL_TEXTURE_EXTERNAL_OES:
  40           return VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
  41         case GL_TEXTURE_RECTANGLE_ARB:
  42           return VideoFrameExternalResources::IO_SURFACE;
  43         default:
  44           NOTREACHED();
  45           break;
  46       }
  47       break;
  48     case media::PIXEL_FORMAT_I420:
  49       return VideoFrameExternalResources::YUV_RESOURCE;
  50       break;
  51     case media::PIXEL_FORMAT_NV12:
  52       DCHECK_EQ(static_cast<uint32_t>(GL_TEXTURE_RECTANGLE_ARB),
  53                 video_frame->mailbox_holder(0).texture_target);
  54       return VideoFrameExternalResources::IO_SURFACE;
  55       break;
  56     case media::PIXEL_FORMAT_YV12:
  57     case media::PIXEL_FORMAT_YV16:
  58     case media::PIXEL_FORMAT_YV24:
  59     case media::PIXEL_FORMAT_YV12A:
  60     case media::PIXEL_FORMAT_NV21:
  61     case media::PIXEL_FORMAT_YUY2:
  62     case media::PIXEL_FORMAT_RGB24:
  63     case media::PIXEL_FORMAT_RGB32:
  64     case media::PIXEL_FORMAT_MJPEG:
  65     case media::PIXEL_FORMAT_UNKNOWN:
  66       break;
  67   }
  68   return VideoFrameExternalResources::NONE;
  69 }
  70
  71 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  72  public:
  73   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl,
  74                                uint32 sync_point)
  75       : gl_(gl), sync_point_(sync_point) {}
  76   ~SyncPointClientImpl() override {}
  77   uint32 InsertSyncPoint() override {
  78     if (sync_point_)
  79       return sync_point_;
  80     return gl_->InsertSyncPointCHROMIUM();
  81   }
  82   void WaitSyncPoint(uint32 sync_point) override {
  83     if (!sync_point)
  84       return;
  85     gl_->WaitSyncPointCHROMIUM(sync_point);
  86     if (sync_point_) {
  87       gl_->WaitSyncPointCHROMIUM(sync_point_);
  88       sync_point_ = 0;
  89     }
  90   }
  91
  92  private:
  93   gpu::gles2::GLES2Interface* gl_;
  94   uint32 sync_point_;
  95 };
  96
  97 }  // namespace
  98
  99 VideoResourceUpdater::PlaneResource::PlaneResource(
 100     unsigned int resource_id,
 101     const gfx::Size& resource_size,
 102     ResourceFormat resource_format,
 103     gpu::Mailbox mailbox)
 104     : resource_id(resource_id),
 105       resource_size(resource_size),
 106       resource_format(resource_format),
 107       mailbox(mailbox),
 108       ref_count(0),
 109       frame_ptr(nullptr),
 110       plane_index(0u) {
 111 }
 112
 113 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
 114     const PlaneResource& plane_resource,
 115     const media::VideoFrame* video_frame,
 116     size_t plane_index) {
 117   return plane_resource.frame_ptr == video_frame &&
 118          plane_resource.plane_index == plane_index &&
 119          plane_resource.timestamp == video_frame->timestamp();
 120 }
 121
 122 void VideoResourceUpdater::SetPlaneResourceUniqueId(
 123     const media::VideoFrame* video_frame,
 124     size_t plane_index,
 125     PlaneResource* plane_resource) {
 126   plane_resource->frame_ptr = video_frame;
 127   plane_resource->plane_index = plane_index;
 128   plane_resource->timestamp = video_frame->timestamp();
 129 }
 130
 131 VideoFrameExternalResources::VideoFrameExternalResources()
 132     : type(NONE), read_lock_fences_enabled(false) {
 133 }
 134
 135 VideoFrameExternalResources::~VideoFrameExternalResources() {}
 136
 137 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
 138                                            ResourceProvider* resource_provider)
 139     : context_provider_(context_provider),
 140       resource_provider_(resource_provider) {
 141 }
 142
 143 VideoResourceUpdater::~VideoResourceUpdater() {
 144   for (const PlaneResource& plane_resource : all_resources_)
 145     resource_provider_->DeleteResource(plane_resource.resource_id);
 146 }
 147
 148 VideoResourceUpdater::ResourceList::iterator
 149 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
 150                                        ResourceFormat format,
 151                                        bool has_mailbox) {
 152   // TODO(danakj): Abstract out hw/sw resource create/delete from
 153   // ResourceProvider and stop using ResourceProvider in this class.
 154   const ResourceId resource_id = resource_provider_->CreateResource(
 155       plane_size, GL_CLAMP_TO_EDGE, ResourceProvider::TEXTURE_HINT_IMMUTABLE,
 156       format);
 157   if (resource_id == 0)
 158     return all_resources_.end();
 159
 160   gpu::Mailbox mailbox;
 161   if (has_mailbox) {
 162     DCHECK(context_provider_);
 163
 164     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 165
 166     gl->GenMailboxCHROMIUM(mailbox.name);
 167     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 168     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 169                                      mailbox.name);
 170   }
 171   all_resources_.push_front(
 172       PlaneResource(resource_id, plane_size, format, mailbox));
 173   return all_resources_.begin();
 174 }
 175
 176 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 177   DCHECK_EQ(resource_it->ref_count, 0);
 178   resource_provider_->DeleteResource(resource_it->resource_id);
 179   all_resources_.erase(resource_it);
 180 }
 181
 182 VideoFrameExternalResources VideoResourceUpdater::
 183     CreateExternalResourcesFromVideoFrame(
 184         const scoped_refptr<media::VideoFrame>& video_frame) {
 185   if (video_frame->format() == media::PIXEL_FORMAT_UNKNOWN)
 186     return VideoFrameExternalResources();
 187   DCHECK(video_frame->HasTextures() || video_frame->IsMappable());
 188   if (video_frame->HasTextures())
 189     return CreateForHardwarePlanes(video_frame);
 190   else
 191     return CreateForSoftwarePlanes(video_frame);
 192 }
 193
 194 // For frames that we receive in software format, determine the dimensions of
 195 // each plane in the frame.
 196 static gfx::Size SoftwarePlaneDimension(
 197     const scoped_refptr<media::VideoFrame>& input_frame,
 198     bool software_compositor,
 199     size_t plane_index) {
 200   if (!software_compositor) {
 201     return media::VideoFrame::PlaneSize(
 202         input_frame->format(), plane_index, input_frame->coded_size());
 203   }
 204   return input_frame->coded_size();
 205 }
 206
 207 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 208     const scoped_refptr<media::VideoFrame>& video_frame) {
 209   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 210   const media::VideoPixelFormat input_frame_format = video_frame->format();
 211
 212 #if defined(VIDEO_HOLE)
 213   if (video_frame->storage_type() == media::VideoFrame::STORAGE_HOLE) {
 214     VideoFrameExternalResources external_resources;
 215     external_resources.type = VideoFrameExternalResources::HOLE;
 216     return external_resources;
 217   }
 218 #endif  // defined(VIDEO_HOLE)
 219
 220   // Only YUV software video frames are supported.
 221   if (!media::IsYuvPlanar(input_frame_format)) {
 222     NOTREACHED() << media::VideoPixelFormatToString(input_frame_format);
 223     return VideoFrameExternalResources();
 224   }
 225
 226   const bool software_compositor = context_provider_ == NULL;
 227
 228   ResourceFormat output_resource_format =
 229       resource_provider_->yuv_resource_format();
 230   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 231
 232   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 233   // conversion here. That involves an extra copy of each frame to a bitmap.
 234   // Obviously, this is suboptimal and should be addressed once ubercompositor
 235   // starts shaping up.
 236   if (software_compositor) {
 237     output_resource_format = kRGBResourceFormat;
 238     output_plane_count = 1;
 239   }
 240
 241   // Drop recycled resources that are the wrong format.
 242   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 243     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 244       DeleteResource(it++);
 245     else
 246       ++it;
 247   }
 248
 249   const int max_resource_size = resource_provider_->max_texture_size();
 250   std::vector<ResourceList::iterator> plane_resources;
 251   for (size_t i = 0; i < output_plane_count; ++i) {
 252     gfx::Size output_plane_resource_size =
 253         SoftwarePlaneDimension(video_frame, software_compositor, i);
 254     if (output_plane_resource_size.IsEmpty() ||
 255         output_plane_resource_size.width() > max_resource_size ||
 256         output_plane_resource_size.height() > max_resource_size) {
 257       break;
 258     }
 259
 260     // Try recycle a previously-allocated resource.
 261     ResourceList::iterator resource_it = all_resources_.end();
 262     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 263       if (it->resource_size == output_plane_resource_size &&
 264           it->resource_format == output_resource_format) {
 265         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 266           // Bingo, we found a resource that already contains the data we are
 267           // planning to put in it. It's safe to reuse it even if
 268           // resource_provider_ holds some references to it, because those
 269           // references are read-only.
 270           resource_it = it;
 271           break;
 272         }
 273
 274         // This extra check is needed because resources backed by SharedMemory
 275         // are not ref-counted, unlike mailboxes. Full discussion in
 276         // codereview.chromium.org/145273021.
 277         const bool in_use =
 278             software_compositor &&
 279             resource_provider_->InUseByConsumer(it->resource_id);
 280         if (it->ref_count == 0 && !in_use) {
 281           // We found a resource with the correct size that we can overwrite.
 282           resource_it = it;
 283         }
 284       }
 285     }
 286
 287     // Check if we need to allocate a new resource.
 288     if (resource_it == all_resources_.end()) {
 289       resource_it =
 290           AllocateResource(output_plane_resource_size, output_resource_format,
 291                            !software_compositor);
 292     }
 293     if (resource_it == all_resources_.end())
 294       break;
 295
 296     ++resource_it->ref_count;
 297     plane_resources.push_back(resource_it);
 298   }
 299
 300   if (plane_resources.size() != output_plane_count) {
 301     // Allocation failed, nothing will be returned so restore reference counts.
 302     for (ResourceList::iterator resource_it : plane_resources)
 303       --resource_it->ref_count;
 304     return VideoFrameExternalResources();
 305   }
 306
 307   VideoFrameExternalResources external_resources;
 308
 309   if (software_compositor) {
 310     DCHECK_EQ(plane_resources.size(), 1u);
 311     PlaneResource& plane_resource = *plane_resources[0];
 312     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 313     DCHECK(plane_resource.mailbox.IsZero());
 314
 315     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 316       // We need to transfer data from |video_frame| to the plane resource.
 317       if (!video_renderer_)
 318         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 319
 320       ResourceProvider::ScopedWriteLockSoftware lock(
 321           resource_provider_, plane_resource.resource_id);
 322       SkCanvas canvas(lock.sk_bitmap());
 323       // This is software path, so canvas and video_frame are always backed
 324       // by software.
 325       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 326       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 327     }
 328
 329     external_resources.software_resources.push_back(plane_resource.resource_id);
 330     external_resources.software_release_callback =
 331         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 332     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 333     return external_resources;
 334   }
 335
 336   for (size_t i = 0; i < plane_resources.size(); ++i) {
 337     PlaneResource& plane_resource = *plane_resources[i];
 338     // Update each plane's resource id with its content.
 339     DCHECK_EQ(plane_resource.resource_format,
 340               resource_provider_->yuv_resource_format());
 341
 342     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 343       // We need to transfer data from |video_frame| to the plane resource.
 344       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 345
 346       // The |resource_size_pixels| is the size of the resource we want to
 347       // upload to.
 348       gfx::Size resource_size_pixels = plane_resource.resource_size;
 349       // The |video_stride_pixels| is the width of the video frame we are
 350       // uploading (including non-frame data to fill in the stride).
 351       int video_stride_pixels = video_frame->stride(i);
 352
 353       size_t bytes_per_row = ResourceUtil::UncheckedWidthInBytes<size_t>(
 354           resource_size_pixels.width(), plane_resource.resource_format);
 355       // Use 4-byte row alignment (OpenGL default) for upload performance.
 356       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 357       size_t upload_image_stride =
 358           MathUtil::UncheckedRoundUp<size_t>(bytes_per_row, 4u);
 359
 360       const uint8_t* pixels;
 361       size_t video_bytes_per_row = ResourceUtil::UncheckedWidthInBytes<size_t>(
 362           video_stride_pixels, plane_resource.resource_format);
 363       if (upload_image_stride == video_bytes_per_row) {
 364         pixels = video_frame->data(i);
 365       } else {
 366         // Avoid malloc for each frame/plane if possible.
 367         size_t needed_size =
 368             upload_image_stride * resource_size_pixels.height();
 369         if (upload_pixels_.size() < needed_size)
 370           upload_pixels_.resize(needed_size);
 371         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 372           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 373           const uint8_t* src =
 374               video_frame->data(i) + (video_bytes_per_row * row);
 375           memcpy(dst, src, bytes_per_row);
 376         }
 377         pixels = &upload_pixels_[0];
 378       }
 379
 380       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 381                                          resource_size_pixels);
 382       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 383     }
 384
 385     external_resources.mailboxes.push_back(
 386         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 387     external_resources.release_callbacks.push_back(
 388         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 389   }
 390
 391   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 392   return external_resources;
 393 }
 394
 395 // static
 396 void VideoResourceUpdater::ReturnTexture(
 397     base::WeakPtr<VideoResourceUpdater> updater,
 398     const scoped_refptr<media::VideoFrame>& video_frame,
 399     uint32 sync_point,
 400     bool lost_resource,
 401     BlockingTaskRunner* main_thread_task_runner) {
 402   // TODO(dshwang) this case should be forwarded to the decoder as lost
 403   // resource.
 404   if (lost_resource || !updater.get())
 405     return;
 406   // Update the release sync point in |video_frame| with |sync_point|
 407   // returned by the compositor and emit a WaitSyncPointCHROMIUM on
 408   // |video_frame|'s previous sync point using the current GL context.
 409   SyncPointClientImpl client(updater->context_provider_->ContextGL(),
 410                              sync_point);
 411   video_frame->UpdateReleaseSyncPoint(&client);
 412 }
 413
 414 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 415     const scoped_refptr<media::VideoFrame>& video_frame) {
 416   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 417   DCHECK(video_frame->HasTextures());
 418   if (!context_provider_)
 419     return VideoFrameExternalResources();
 420
 421   VideoFrameExternalResources external_resources;
 422   external_resources.read_lock_fences_enabled = true;
 423
 424   external_resources.type = ResourceTypeForVideoFrame(video_frame.get());
 425   if (external_resources.type == VideoFrameExternalResources::NONE) {
 426     DLOG(ERROR) << "Unsupported Texture format"
 427                 << media::VideoPixelFormatToString(video_frame->format());
 428     return external_resources;
 429   }
 430
 431   const size_t num_planes = media::VideoFrame::NumPlanes(video_frame->format());
 432   for (size_t i = 0; i < num_planes; ++i) {
 433     const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
 434     if (mailbox_holder.mailbox.IsZero())
 435       break;
 436     external_resources.mailboxes.push_back(
 437         TextureMailbox(mailbox_holder.mailbox, mailbox_holder.texture_target,
 438                        mailbox_holder.sync_point, video_frame->coded_size(),
 439                        video_frame->metadata()->IsTrue(
 440                            media::VideoFrameMetadata::ALLOW_OVERLAY)));
 441     external_resources.release_callbacks.push_back(
 442         base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 443   }
 444   return external_resources;
 445 }
 446
 447 // static
 448 void VideoResourceUpdater::RecycleResource(
 449     base::WeakPtr<VideoResourceUpdater> updater,
 450     ResourceId resource_id,
 451     uint32 sync_point,
 452     bool lost_resource,
 453     BlockingTaskRunner* main_thread_task_runner) {
 454   if (!updater.get()) {
 455     // Resource was already deleted.
 456     return;
 457   }
 458
 459   const ResourceList::iterator resource_it = std::find_if(
 460       updater->all_resources_.begin(), updater->all_resources_.end(),
 461       [resource_id](const PlaneResource& plane_resource) {
 462         return plane_resource.resource_id == resource_id;
 463       });
 464   if (resource_it == updater->all_resources_.end())
 465     return;
 466
 467   ContextProvider* context_provider = updater->context_provider_;
 468   if (context_provider && sync_point) {
 469     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 470   }
 471
 472   if (lost_resource) {
 473     resource_it->ref_count = 0;
 474     updater->DeleteResource(resource_it);
 475     return;
 476   }
 477
 478   --resource_it->ref_count;
 479   DCHECK_GE(resource_it->ref_count, 0);
 480 }
 481
 482 }  // namespace cc