cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  29  public:
  30   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl,
  31                                uint32 sync_point)
  32       : gl_(gl), sync_point_(sync_point) {}
  33   ~SyncPointClientImpl() override {}
  34   uint32 InsertSyncPoint() override {
  35     if (sync_point_)
  36       return sync_point_;
  37     return gl_->InsertSyncPointCHROMIUM();
  38   }
  39   void WaitSyncPoint(uint32 sync_point) override {
  40     if (!sync_point)
  41       return;
  42     gl_->WaitSyncPointCHROMIUM(sync_point);
  43     if (sync_point_) {
  44       gl_->WaitSyncPointCHROMIUM(sync_point_);
  45       sync_point_ = 0;
  46     }
  47   }
  48
  49  private:
  50   gpu::gles2::GLES2Interface* gl_;
  51   uint32 sync_point_;
  52 };
  53
  54 }  // namespace
  55
  56 VideoResourceUpdater::PlaneResource::PlaneResource(
  57     unsigned int resource_id,
  58     const gfx::Size& resource_size,
  59     ResourceFormat resource_format,
  60     gpu::Mailbox mailbox)
  61     : resource_id(resource_id),
  62       resource_size(resource_size),
  63       resource_format(resource_format),
  64       mailbox(mailbox),
  65       ref_count(0),
  66       frame_ptr(nullptr),
  67       plane_index(0) {
  68 }
  69
  70 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
  71     const PlaneResource& plane_resource,
  72     const media::VideoFrame* video_frame,
  73     int plane_index) {
  74   return plane_resource.frame_ptr == video_frame &&
  75          plane_resource.plane_index == plane_index &&
  76          plane_resource.timestamp == video_frame->timestamp();
  77 }
  78
  79 void VideoResourceUpdater::SetPlaneResourceUniqueId(
  80     const media::VideoFrame* video_frame,
  81     int plane_index,
  82     PlaneResource* plane_resource) {
  83   plane_resource->frame_ptr = video_frame;
  84   plane_resource->plane_index = plane_index;
  85   plane_resource->timestamp = video_frame->timestamp();
  86 }
  87
  88 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  89
  90 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  91
  92 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  93                                            ResourceProvider* resource_provider)
  94     : context_provider_(context_provider),
  95       resource_provider_(resource_provider) {
  96 }
  97
  98 VideoResourceUpdater::~VideoResourceUpdater() {
  99   for (const PlaneResource& plane_resource : all_resources_)
 100     resource_provider_->DeleteResource(plane_resource.resource_id);
 101 }
 102
 103 VideoResourceUpdater::ResourceList::iterator
 104 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
 105                                        ResourceFormat format,
 106                                        bool has_mailbox) {
 107   // TODO(danakj): Abstract out hw/sw resource create/delete from
 108   // ResourceProvider and stop using ResourceProvider in this class.
 109   const ResourceProvider::ResourceId resource_id =
 110       resource_provider_->CreateResource(
 111           plane_size, GL_CLAMP_TO_EDGE,
 112           ResourceProvider::TEXTURE_HINT_IMMUTABLE, format);
 113   if (resource_id == 0)
 114     return all_resources_.end();
 115
 116   gpu::Mailbox mailbox;
 117   if (has_mailbox) {
 118     DCHECK(context_provider_);
 119
 120     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 121
 122     gl->GenMailboxCHROMIUM(mailbox.name);
 123     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 124     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 125                                      mailbox.name);
 126   }
 127   all_resources_.push_front(
 128       PlaneResource(resource_id, plane_size, format, mailbox));
 129   return all_resources_.begin();
 130 }
 131
 132 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 133   DCHECK_EQ(resource_it->ref_count, 0);
 134   resource_provider_->DeleteResource(resource_it->resource_id);
 135   all_resources_.erase(resource_it);
 136 }
 137
 138 VideoFrameExternalResources VideoResourceUpdater::
 139     CreateExternalResourcesFromVideoFrame(
 140         const scoped_refptr<media::VideoFrame>& video_frame) {
 141   if (!VerifyFrame(video_frame))
 142     return VideoFrameExternalResources();
 143
 144   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
 145     return CreateForHardwarePlanes(video_frame);
 146   else
 147     return CreateForSoftwarePlanes(video_frame);
 148 }
 149
 150 bool VideoResourceUpdater::VerifyFrame(
 151     const scoped_refptr<media::VideoFrame>& video_frame) {
 152   switch (video_frame->format()) {
 153     // Acceptable inputs.
 154     case media::VideoFrame::YV12:
 155     case media::VideoFrame::I420:
 156     case media::VideoFrame::YV12A:
 157     case media::VideoFrame::YV16:
 158     case media::VideoFrame::YV12J:
 159     case media::VideoFrame::YV12HD:
 160     case media::VideoFrame::YV24:
 161     case media::VideoFrame::NATIVE_TEXTURE:
 162 #if defined(VIDEO_HOLE)
 163     case media::VideoFrame::HOLE:
 164 #endif  // defined(VIDEO_HOLE)
 165     case media::VideoFrame::ARGB:
 166       return true;
 167
 168     // Unacceptable inputs. ¯\(°_o)/¯
 169     case media::VideoFrame::UNKNOWN:
 170     case media::VideoFrame::NV12:
 171       break;
 172   }
 173   return false;
 174 }
 175
 176 // For frames that we receive in software format, determine the dimensions of
 177 // each plane in the frame.
 178 static gfx::Size SoftwarePlaneDimension(
 179     const scoped_refptr<media::VideoFrame>& input_frame,
 180     bool software_compositor,
 181     size_t plane_index) {
 182   if (!software_compositor) {
 183     return media::VideoFrame::PlaneSize(
 184         input_frame->format(), plane_index, input_frame->coded_size());
 185   }
 186   return input_frame->coded_size();
 187 }
 188
 189 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 190     const scoped_refptr<media::VideoFrame>& video_frame) {
 191   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 192   media::VideoFrame::Format input_frame_format = video_frame->format();
 193
 194 #if defined(VIDEO_HOLE)
 195   if (input_frame_format == media::VideoFrame::HOLE) {
 196     VideoFrameExternalResources external_resources;
 197     external_resources.type = VideoFrameExternalResources::HOLE;
 198     return external_resources;
 199   }
 200 #endif  // defined(VIDEO_HOLE)
 201
 202   // Only YUV software video frames are supported.
 203   if (input_frame_format != media::VideoFrame::YV12 &&
 204       input_frame_format != media::VideoFrame::I420 &&
 205       input_frame_format != media::VideoFrame::YV12A &&
 206       input_frame_format != media::VideoFrame::YV12J &&
 207       input_frame_format != media::VideoFrame::YV12HD &&
 208       input_frame_format != media::VideoFrame::YV16 &&
 209       input_frame_format != media::VideoFrame::YV24) {
 210     NOTREACHED() << input_frame_format;
 211     return VideoFrameExternalResources();
 212   }
 213
 214   bool software_compositor = context_provider_ == NULL;
 215
 216   ResourceFormat output_resource_format =
 217       resource_provider_->yuv_resource_format();
 218   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 219
 220   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 221   // conversion here. That involves an extra copy of each frame to a bitmap.
 222   // Obviously, this is suboptimal and should be addressed once ubercompositor
 223   // starts shaping up.
 224   if (software_compositor) {
 225     output_resource_format = kRGBResourceFormat;
 226     output_plane_count = 1;
 227   }
 228
 229   // Drop recycled resources that are the wrong format.
 230   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 231     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 232       DeleteResource(it++);
 233     else
 234       ++it;
 235   }
 236
 237   const int max_resource_size = resource_provider_->max_texture_size();
 238   std::vector<ResourceList::iterator> plane_resources;
 239   for (size_t i = 0; i < output_plane_count; ++i) {
 240     gfx::Size output_plane_resource_size =
 241         SoftwarePlaneDimension(video_frame, software_compositor, i);
 242     if (output_plane_resource_size.IsEmpty() ||
 243         output_plane_resource_size.width() > max_resource_size ||
 244         output_plane_resource_size.height() > max_resource_size) {
 245       break;
 246     }
 247
 248     // Try recycle a previously-allocated resource.
 249     ResourceList::iterator resource_it = all_resources_.end();
 250     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 251       if (it->resource_size == output_plane_resource_size &&
 252           it->resource_format == output_resource_format) {
 253         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 254           // Bingo, we found a resource that already contains the data we are
 255           // planning to put in it. It's safe to reuse it even if
 256           // resource_provider_ holds some references to it, because those
 257           // references are read-only.
 258           resource_it = it;
 259           break;
 260         }
 261
 262         // This extra check is needed because resources backed by SharedMemory
 263         // are not ref-counted, unlike mailboxes. Full discussion in
 264         // codereview.chromium.org/145273021.
 265         const bool in_use =
 266             software_compositor &&
 267             resource_provider_->InUseByConsumer(it->resource_id);
 268         if (it->ref_count == 0 && !in_use) {
 269           // We found a resource with the correct size that we can overwrite.
 270           resource_it = it;
 271         }
 272       }
 273     }
 274
 275     // Check if we need to allocate a new resource.
 276     if (resource_it == all_resources_.end()) {
 277       resource_it =
 278           AllocateResource(output_plane_resource_size, output_resource_format,
 279                            !software_compositor);
 280     }
 281     if (resource_it == all_resources_.end())
 282       break;
 283
 284     ++resource_it->ref_count;
 285     plane_resources.push_back(resource_it);
 286   }
 287
 288   if (plane_resources.size() != output_plane_count) {
 289     // Allocation failed, nothing will be returned so restore reference counts.
 290     for (ResourceList::iterator resource_it : plane_resources)
 291       --resource_it->ref_count;
 292     return VideoFrameExternalResources();
 293   }
 294
 295   VideoFrameExternalResources external_resources;
 296
 297   if (software_compositor) {
 298     DCHECK_EQ(plane_resources.size(), 1u);
 299     PlaneResource& plane_resource = *plane_resources[0];
 300     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 301     DCHECK(plane_resource.mailbox.IsZero());
 302
 303     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 304       // We need to transfer data from |video_frame| to the plane resource.
 305       if (!video_renderer_)
 306         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 307
 308       ResourceProvider::ScopedWriteLockSoftware lock(
 309           resource_provider_, plane_resource.resource_id);
 310       SkCanvas canvas(lock.sk_bitmap());
 311       // This is software path, so canvas and video_frame are always backed
 312       // by software.
 313       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 314       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 315     }
 316
 317     external_resources.software_resources.push_back(plane_resource.resource_id);
 318     external_resources.software_release_callback =
 319         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 320     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 321     return external_resources;
 322   }
 323
 324   for (size_t i = 0; i < plane_resources.size(); ++i) {
 325     PlaneResource& plane_resource = *plane_resources[i];
 326     // Update each plane's resource id with its content.
 327     DCHECK_EQ(plane_resource.resource_format,
 328               resource_provider_->yuv_resource_format());
 329
 330     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 331       // We need to transfer data from |video_frame| to the plane resource.
 332       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 333
 334       // The |resource_size_pixels| is the size of the resource we want to
 335       // upload to.
 336       gfx::Size resource_size_pixels = plane_resource.resource_size;
 337       // The |video_stride_pixels| is the width of the video frame we are
 338       // uploading (including non-frame data to fill in the stride).
 339       size_t video_stride_pixels = video_frame->stride(i);
 340
 341       size_t bytes_per_pixel = BitsPerPixel(plane_resource.resource_format) / 8;
 342       // Use 4-byte row alignment (OpenGL default) for upload performance.
 343       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 344       size_t upload_image_stride =
 345           RoundUp<size_t>(bytes_per_pixel * resource_size_pixels.width(), 4u);
 346
 347       const uint8_t* pixels;
 348       if (upload_image_stride == video_stride_pixels * bytes_per_pixel) {
 349         pixels = video_frame->data(i);
 350       } else {
 351         // Avoid malloc for each frame/plane if possible.
 352         size_t needed_size =
 353             upload_image_stride * resource_size_pixels.height();
 354         if (upload_pixels_.size() < needed_size)
 355           upload_pixels_.resize(needed_size);
 356         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 357           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 358           const uint8_t* src = video_frame->data(i) +
 359                                bytes_per_pixel * video_stride_pixels * row;
 360           memcpy(dst, src, resource_size_pixels.width() * bytes_per_pixel);
 361         }
 362         pixels = &upload_pixels_[0];
 363       }
 364
 365       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 366                                          resource_size_pixels);
 367       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 368     }
 369
 370     external_resources.mailboxes.push_back(
 371         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 372     external_resources.release_callbacks.push_back(
 373         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 374   }
 375
 376   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 377   return external_resources;
 378 }
 379
 380 // static
 381 void VideoResourceUpdater::ReturnTexture(
 382     base::WeakPtr<VideoResourceUpdater> updater,
 383     const scoped_refptr<media::VideoFrame>& video_frame,
 384     uint32 sync_point,
 385     bool lost_resource,
 386     BlockingTaskRunner* main_thread_task_runner) {
 387   // TODO(dshwang) this case should be forwarded to the decoder as lost
 388   // resource.
 389   if (lost_resource || !updater.get())
 390     return;
 391   // Update the release sync point in |video_frame| with |sync_point|
 392   // returned by the compositor and emit a WaitSyncPointCHROMIUM on
 393   // |video_frame|'s previous sync point using the current GL context.
 394   SyncPointClientImpl client(updater->context_provider_->ContextGL(),
 395                              sync_point);
 396   video_frame->UpdateReleaseSyncPoint(&client);
 397 }
 398
 399 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 400     const scoped_refptr<media::VideoFrame>& video_frame) {
 401   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 402   media::VideoFrame::Format frame_format = video_frame->format();
 403
 404   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
 405   if (!context_provider_)
 406     return VideoFrameExternalResources();
 407
 408   size_t textures =
 409       media::VideoFrame::NumTextures(video_frame->texture_format());
 410   DCHECK_GE(textures, 1u);
 411   VideoFrameExternalResources external_resources;
 412   switch (video_frame->texture_format()) {
 413     case media::VideoFrame::TEXTURE_RGBA:
 414       DCHECK_EQ(1u, textures);
 415       switch (video_frame->mailbox_holder(0).texture_target) {
 416         case GL_TEXTURE_2D:
 417           external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
 418           break;
 419         case GL_TEXTURE_EXTERNAL_OES:
 420           external_resources.type =
 421               VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 422           break;
 423         case GL_TEXTURE_RECTANGLE_ARB:
 424           external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 425           break;
 426         default:
 427           NOTREACHED();
 428           return VideoFrameExternalResources();
 429       }
 430       break;
 431     case media::VideoFrame::TEXTURE_YUV_420:
 432       external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 433       break;
 434   }
 435   DCHECK_NE(VideoFrameExternalResources::NONE, external_resources.type);
 436
 437   for (size_t i = 0; i < textures; ++i) {
 438     const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
 439     external_resources.mailboxes.push_back(
 440         TextureMailbox(mailbox_holder.mailbox, mailbox_holder.texture_target,
 441                        mailbox_holder.sync_point));
 442     external_resources.mailboxes.back().set_allow_overlay(
 443         video_frame->allow_overlay());
 444     external_resources.release_callbacks.push_back(
 445         base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 446   }
 447   return external_resources;
 448 }
 449
 450 // static
 451 void VideoResourceUpdater::RecycleResource(
 452     base::WeakPtr<VideoResourceUpdater> updater,
 453     ResourceProvider::ResourceId resource_id,
 454     uint32 sync_point,
 455     bool lost_resource,
 456     BlockingTaskRunner* main_thread_task_runner) {
 457   if (!updater.get()) {
 458     // Resource was already deleted.
 459     return;
 460   }
 461
 462   const ResourceList::iterator resource_it = std::find_if(
 463       updater->all_resources_.begin(), updater->all_resources_.end(),
 464       [resource_id](const PlaneResource& plane_resource) {
 465         return plane_resource.resource_id == resource_id;
 466       });
 467   if (resource_it == updater->all_resources_.end())
 468     return;
 469
 470   ContextProvider* context_provider = updater->context_provider_;
 471   if (context_provider && sync_point) {
 472     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 473   }
 474
 475   if (lost_resource) {
 476     resource_it->ref_count = 0;
 477     updater->DeleteResource(resource_it);
 478     return;
 479   }
 480
 481   --resource_it->ref_count;
 482   DCHECK_GE(resource_it->ref_count, 0);
 483 }
 484
 485 }  // namespace cc