cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/math_util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 VideoFrameExternalResources::ResourceType ResourceTypeForVideoFrame(
  29     media::VideoFrame* video_frame) {
  30   switch (video_frame->format()) {
  31     case media::PIXEL_FORMAT_ARGB:
  32     case media::PIXEL_FORMAT_XRGB:
  33     case media::PIXEL_FORMAT_UYVY:
  34       switch (video_frame->mailbox_holder(0).texture_target) {
  35         case GL_TEXTURE_2D:
  36           return (video_frame->format() == media::PIXEL_FORMAT_XRGB)
  37                      ? VideoFrameExternalResources::RGB_RESOURCE
  38                      : VideoFrameExternalResources::RGBA_RESOURCE;
  39         case GL_TEXTURE_EXTERNAL_OES:
  40           return VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
  41         case GL_TEXTURE_RECTANGLE_ARB:
  42           return VideoFrameExternalResources::IO_SURFACE;
  43         default:
  44           NOTREACHED();
  45           break;
  46       }
  47       break;
  48     case media::PIXEL_FORMAT_I420:
  49       return VideoFrameExternalResources::YUV_RESOURCE;
  50       break;
  51     case media::PIXEL_FORMAT_NV12:
  52       DCHECK_EQ(static_cast<uint32_t>(GL_TEXTURE_RECTANGLE_ARB),
  53                 video_frame->mailbox_holder(0).texture_target);
  54       return VideoFrameExternalResources::IO_SURFACE;
  55       break;
  56     case media::PIXEL_FORMAT_YV12:
  57     case media::PIXEL_FORMAT_YV16:
  58     case media::PIXEL_FORMAT_YV24:
  59     case media::PIXEL_FORMAT_YV12A:
  60     case media::PIXEL_FORMAT_NV21:
  61     case media::PIXEL_FORMAT_YUY2:
  62     case media::PIXEL_FORMAT_RGB24:
  63     case media::PIXEL_FORMAT_RGB32:
  64     case media::PIXEL_FORMAT_MJPEG:
  65     case media::PIXEL_FORMAT_MT21:
  66     case media::PIXEL_FORMAT_UNKNOWN:
  67       break;
  68   }
  69   return VideoFrameExternalResources::NONE;
  70 }
  71
  72 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  73  public:
  74   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl,
  75                                uint32 sync_point)
  76       : gl_(gl), sync_point_(sync_point) {}
  77   ~SyncPointClientImpl() override {}
  78   uint32 InsertSyncPoint() override {
  79     if (sync_point_)
  80       return sync_point_;
  81     return gl_->InsertSyncPointCHROMIUM();
  82   }
  83   void WaitSyncPoint(uint32 sync_point) override {
  84     if (!sync_point)
  85       return;
  86     gl_->WaitSyncPointCHROMIUM(sync_point);
  87     if (sync_point_) {
  88       gl_->WaitSyncPointCHROMIUM(sync_point_);
  89       sync_point_ = 0;
  90     }
  91   }
  92
  93  private:
  94   gpu::gles2::GLES2Interface* gl_;
  95   uint32 sync_point_;
  96 };
  97
  98 }  // namespace
  99
 100 VideoResourceUpdater::PlaneResource::PlaneResource(
 101     unsigned int resource_id,
 102     const gfx::Size& resource_size,
 103     ResourceFormat resource_format,
 104     gpu::Mailbox mailbox)
 105     : resource_id(resource_id),
 106       resource_size(resource_size),
 107       resource_format(resource_format),
 108       mailbox(mailbox),
 109       ref_count(0),
 110       frame_ptr(nullptr),
 111       plane_index(0u) {
 112 }
 113
 114 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
 115     const PlaneResource& plane_resource,
 116     const media::VideoFrame* video_frame,
 117     size_t plane_index) {
 118   return plane_resource.frame_ptr == video_frame &&
 119          plane_resource.plane_index == plane_index &&
 120          plane_resource.timestamp == video_frame->timestamp();
 121 }
 122
 123 void VideoResourceUpdater::SetPlaneResourceUniqueId(
 124     const media::VideoFrame* video_frame,
 125     size_t plane_index,
 126     PlaneResource* plane_resource) {
 127   plane_resource->frame_ptr = video_frame;
 128   plane_resource->plane_index = plane_index;
 129   plane_resource->timestamp = video_frame->timestamp();
 130 }
 131
 132 VideoFrameExternalResources::VideoFrameExternalResources()
 133     : type(NONE), read_lock_fences_enabled(false) {
 134 }
 135
 136 VideoFrameExternalResources::~VideoFrameExternalResources() {}
 137
 138 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
 139                                            ResourceProvider* resource_provider)
 140     : context_provider_(context_provider),
 141       resource_provider_(resource_provider) {
 142 }
 143
 144 VideoResourceUpdater::~VideoResourceUpdater() {
 145   for (const PlaneResource& plane_resource : all_resources_)
 146     resource_provider_->DeleteResource(plane_resource.resource_id);
 147 }
 148
 149 VideoResourceUpdater::ResourceList::iterator
 150 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
 151                                        ResourceFormat format,
 152                                        bool has_mailbox) {
 153   // TODO(danakj): Abstract out hw/sw resource create/delete from
 154   // ResourceProvider and stop using ResourceProvider in this class.
 155   const ResourceId resource_id = resource_provider_->CreateResource(
 156       plane_size, GL_CLAMP_TO_EDGE, ResourceProvider::TEXTURE_HINT_IMMUTABLE,
 157       format);
 158   if (resource_id == 0)
 159     return all_resources_.end();
 160
 161   gpu::Mailbox mailbox;
 162   if (has_mailbox) {
 163     DCHECK(context_provider_);
 164
 165     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 166
 167     gl->GenMailboxCHROMIUM(mailbox.name);
 168     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 169     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 170                                      mailbox.name);
 171   }
 172   all_resources_.push_front(
 173       PlaneResource(resource_id, plane_size, format, mailbox));
 174   return all_resources_.begin();
 175 }
 176
 177 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 178   DCHECK_EQ(resource_it->ref_count, 0);
 179   resource_provider_->DeleteResource(resource_it->resource_id);
 180   all_resources_.erase(resource_it);
 181 }
 182
 183 VideoFrameExternalResources VideoResourceUpdater::
 184     CreateExternalResourcesFromVideoFrame(
 185         const scoped_refptr<media::VideoFrame>& video_frame) {
 186   if (video_frame->format() == media::PIXEL_FORMAT_UNKNOWN)
 187     return VideoFrameExternalResources();
 188   DCHECK(video_frame->HasTextures() || video_frame->IsMappable());
 189   if (video_frame->HasTextures())
 190     return CreateForHardwarePlanes(video_frame);
 191   else
 192     return CreateForSoftwarePlanes(video_frame);
 193 }
 194
 195 // For frames that we receive in software format, determine the dimensions of
 196 // each plane in the frame.
 197 static gfx::Size SoftwarePlaneDimension(
 198     const scoped_refptr<media::VideoFrame>& input_frame,
 199     bool software_compositor,
 200     size_t plane_index) {
 201   if (!software_compositor) {
 202     return media::VideoFrame::PlaneSize(
 203         input_frame->format(), plane_index, input_frame->coded_size());
 204   }
 205   return input_frame->coded_size();
 206 }
 207
 208 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 209     const scoped_refptr<media::VideoFrame>& video_frame) {
 210   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 211   const media::VideoPixelFormat input_frame_format = video_frame->format();
 212
 213 #if defined(VIDEO_HOLE)
 214   if (video_frame->storage_type() == media::VideoFrame::STORAGE_HOLE) {
 215     VideoFrameExternalResources external_resources;
 216     external_resources.type = VideoFrameExternalResources::HOLE;
 217     return external_resources;
 218   }
 219 #endif  // defined(VIDEO_HOLE)
 220
 221   // Only YUV software video frames are supported.
 222   if (!media::IsYuvPlanar(input_frame_format)) {
 223     NOTREACHED() << media::VideoPixelFormatToString(input_frame_format);
 224     return VideoFrameExternalResources();
 225   }
 226
 227   const bool software_compositor = context_provider_ == NULL;
 228
 229   ResourceFormat output_resource_format =
 230       resource_provider_->yuv_resource_format();
 231   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 232
 233   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 234   // conversion here. That involves an extra copy of each frame to a bitmap.
 235   // Obviously, this is suboptimal and should be addressed once ubercompositor
 236   // starts shaping up.
 237   if (software_compositor) {
 238     output_resource_format = kRGBResourceFormat;
 239     output_plane_count = 1;
 240   }
 241
 242   // Drop recycled resources that are the wrong format.
 243   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 244     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 245       DeleteResource(it++);
 246     else
 247       ++it;
 248   }
 249
 250   const int max_resource_size = resource_provider_->max_texture_size();
 251   std::vector<ResourceList::iterator> plane_resources;
 252   for (size_t i = 0; i < output_plane_count; ++i) {
 253     gfx::Size output_plane_resource_size =
 254         SoftwarePlaneDimension(video_frame, software_compositor, i);
 255     if (output_plane_resource_size.IsEmpty() ||
 256         output_plane_resource_size.width() > max_resource_size ||
 257         output_plane_resource_size.height() > max_resource_size) {
 258       break;
 259     }
 260
 261     // Try recycle a previously-allocated resource.
 262     ResourceList::iterator resource_it = all_resources_.end();
 263     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 264       if (it->resource_size == output_plane_resource_size &&
 265           it->resource_format == output_resource_format) {
 266         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 267           // Bingo, we found a resource that already contains the data we are
 268           // planning to put in it. It's safe to reuse it even if
 269           // resource_provider_ holds some references to it, because those
 270           // references are read-only.
 271           resource_it = it;
 272           break;
 273         }
 274
 275         // This extra check is needed because resources backed by SharedMemory
 276         // are not ref-counted, unlike mailboxes. Full discussion in
 277         // codereview.chromium.org/145273021.
 278         const bool in_use =
 279             software_compositor &&
 280             resource_provider_->InUseByConsumer(it->resource_id);
 281         if (it->ref_count == 0 && !in_use) {
 282           // We found a resource with the correct size that we can overwrite.
 283           resource_it = it;
 284         }
 285       }
 286     }
 287
 288     // Check if we need to allocate a new resource.
 289     if (resource_it == all_resources_.end()) {
 290       resource_it =
 291           AllocateResource(output_plane_resource_size, output_resource_format,
 292                            !software_compositor);
 293     }
 294     if (resource_it == all_resources_.end())
 295       break;
 296
 297     ++resource_it->ref_count;
 298     plane_resources.push_back(resource_it);
 299   }
 300
 301   if (plane_resources.size() != output_plane_count) {
 302     // Allocation failed, nothing will be returned so restore reference counts.
 303     for (ResourceList::iterator resource_it : plane_resources)
 304       --resource_it->ref_count;
 305     return VideoFrameExternalResources();
 306   }
 307
 308   VideoFrameExternalResources external_resources;
 309
 310   if (software_compositor) {
 311     DCHECK_EQ(plane_resources.size(), 1u);
 312     PlaneResource& plane_resource = *plane_resources[0];
 313     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 314     DCHECK(plane_resource.mailbox.IsZero());
 315
 316     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 317       // We need to transfer data from |video_frame| to the plane resource.
 318       if (!video_renderer_)
 319         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 320
 321       ResourceProvider::ScopedWriteLockSoftware lock(
 322           resource_provider_, plane_resource.resource_id);
 323       SkCanvas canvas(lock.sk_bitmap());
 324       // This is software path, so canvas and video_frame are always backed
 325       // by software.
 326       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 327       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 328     }
 329
 330     external_resources.software_resources.push_back(plane_resource.resource_id);
 331     external_resources.software_release_callback =
 332         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 333     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 334     return external_resources;
 335   }
 336
 337   for (size_t i = 0; i < plane_resources.size(); ++i) {
 338     PlaneResource& plane_resource = *plane_resources[i];
 339     // Update each plane's resource id with its content.
 340     DCHECK_EQ(plane_resource.resource_format,
 341               resource_provider_->yuv_resource_format());
 342
 343     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 344       // We need to transfer data from |video_frame| to the plane resource.
 345       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 346
 347       // The |resource_size_pixels| is the size of the resource we want to
 348       // upload to.
 349       gfx::Size resource_size_pixels = plane_resource.resource_size;
 350       // The |video_stride_pixels| is the width of the video frame we are
 351       // uploading (including non-frame data to fill in the stride).
 352       int video_stride_pixels = video_frame->stride(i);
 353
 354       size_t bytes_per_row = ResourceUtil::UncheckedWidthInBytes<size_t>(
 355           resource_size_pixels.width(), plane_resource.resource_format);
 356       // Use 4-byte row alignment (OpenGL default) for upload performance.
 357       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 358       size_t upload_image_stride =
 359           MathUtil::UncheckedRoundUp<size_t>(bytes_per_row, 4u);
 360
 361       const uint8_t* pixels;
 362       size_t video_bytes_per_row = ResourceUtil::UncheckedWidthInBytes<size_t>(
 363           video_stride_pixels, plane_resource.resource_format);
 364       if (upload_image_stride == video_bytes_per_row) {
 365         pixels = video_frame->data(i);
 366       } else {
 367         // Avoid malloc for each frame/plane if possible.
 368         size_t needed_size =
 369             upload_image_stride * resource_size_pixels.height();
 370         if (upload_pixels_.size() < needed_size)
 371           upload_pixels_.resize(needed_size);
 372         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 373           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 374           const uint8_t* src =
 375               video_frame->data(i) + (video_bytes_per_row * row);
 376           memcpy(dst, src, bytes_per_row);
 377         }
 378         pixels = &upload_pixels_[0];
 379       }
 380
 381       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 382                                          resource_size_pixels);
 383       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 384     }
 385
 386     external_resources.mailboxes.push_back(
 387         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 388     external_resources.release_callbacks.push_back(
 389         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 390   }
 391
 392   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 393   return external_resources;
 394 }
 395
 396 // static
 397 void VideoResourceUpdater::ReturnTexture(
 398     base::WeakPtr<VideoResourceUpdater> updater,
 399     const scoped_refptr<media::VideoFrame>& video_frame,
 400     uint32 sync_point,
 401     bool lost_resource,
 402     BlockingTaskRunner* main_thread_task_runner) {
 403   // TODO(dshwang) this case should be forwarded to the decoder as lost
 404   // resource.
 405   if (lost_resource || !updater.get())
 406     return;
 407   // Update the release sync point in |video_frame| with |sync_point|
 408   // returned by the compositor and emit a WaitSyncPointCHROMIUM on
 409   // |video_frame|'s previous sync point using the current GL context.
 410   SyncPointClientImpl client(updater->context_provider_->ContextGL(),
 411                              sync_point);
 412   video_frame->UpdateReleaseSyncPoint(&client);
 413 }
 414
 415 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 416     const scoped_refptr<media::VideoFrame>& video_frame) {
 417   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 418   DCHECK(video_frame->HasTextures());
 419   if (!context_provider_)
 420     return VideoFrameExternalResources();
 421
 422   VideoFrameExternalResources external_resources;
 423   external_resources.read_lock_fences_enabled = true;
 424
 425   external_resources.type = ResourceTypeForVideoFrame(video_frame.get());
 426   if (external_resources.type == VideoFrameExternalResources::NONE) {
 427     DLOG(ERROR) << "Unsupported Texture format"
 428                 << media::VideoPixelFormatToString(video_frame->format());
 429     return external_resources;
 430   }
 431
 432   const size_t num_planes = media::VideoFrame::NumPlanes(video_frame->format());
 433   for (size_t i = 0; i < num_planes; ++i) {
 434     const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
 435     if (mailbox_holder.mailbox.IsZero())
 436       break;
 437     external_resources.mailboxes.push_back(
 438         TextureMailbox(mailbox_holder.mailbox, mailbox_holder.texture_target,
 439                        mailbox_holder.sync_point, video_frame->coded_size(),
 440                        video_frame->metadata()->IsTrue(
 441                            media::VideoFrameMetadata::ALLOW_OVERLAY)));
 442     external_resources.release_callbacks.push_back(
 443         base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 444   }
 445   return external_resources;
 446 }
 447
 448 // static
 449 void VideoResourceUpdater::RecycleResource(
 450     base::WeakPtr<VideoResourceUpdater> updater,
 451     ResourceId resource_id,
 452     uint32 sync_point,
 453     bool lost_resource,
 454     BlockingTaskRunner* main_thread_task_runner) {
 455   if (!updater.get()) {
 456     // Resource was already deleted.
 457     return;
 458   }
 459
 460   const ResourceList::iterator resource_it = std::find_if(
 461       updater->all_resources_.begin(), updater->all_resources_.end(),
 462       [resource_id](const PlaneResource& plane_resource) {
 463         return plane_resource.resource_id == resource_id;
 464       });
 465   if (resource_it == updater->all_resources_.end())
 466     return;
 467
 468   ContextProvider* context_provider = updater->context_provider_;
 469   if (context_provider && sync_point) {
 470     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 471   }
 472
 473   if (lost_resource) {
 474     resource_it->ref_count = 0;
 475     updater->DeleteResource(resource_it);
 476     return;
 477   }
 478
 479   --resource_it->ref_count;
 480   DCHECK_GE(resource_it->ref_count, 0);
 481 }
 482
 483 }  // namespace cc