cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include <algorithm>
   8
   9 #include "base/bind.h"
  10 #include "base/trace_event/trace_event.h"
  11 #include "cc/base/util.h"
  12 #include "cc/output/gl_renderer.h"
  13 #include "cc/resources/resource_provider.h"
  14 #include "gpu/GLES2/gl2extchromium.h"
  15 #include "gpu/command_buffer/client/gles2_interface.h"
  16 #include "media/base/video_frame.h"
  17 #include "media/blink/skcanvas_video_renderer.h"
  18 #include "third_party/khronos/GLES2/gl2.h"
  19 #include "third_party/khronos/GLES2/gl2ext.h"
  20 #include "ui/gfx/geometry/size_conversions.h"
  21
  22 namespace cc {
  23
  24 namespace {
  25
  26 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  27
  28 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  29  public:
  30   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
  31   ~SyncPointClientImpl() override {}
  32   uint32 InsertSyncPoint() override { return gl_->InsertSyncPointCHROMIUM(); }
  33   void WaitSyncPoint(uint32 sync_point) override {
  34     gl_->WaitSyncPointCHROMIUM(sync_point);
  35   }
  36
  37  private:
  38   gpu::gles2::GLES2Interface* gl_;
  39 };
  40
  41 }  // namespace
  42
  43 VideoResourceUpdater::PlaneResource::PlaneResource(
  44     unsigned int resource_id,
  45     const gfx::Size& resource_size,
  46     ResourceFormat resource_format,
  47     gpu::Mailbox mailbox)
  48     : resource_id(resource_id),
  49       resource_size(resource_size),
  50       resource_format(resource_format),
  51       mailbox(mailbox),
  52       ref_count(0),
  53       frame_ptr(nullptr),
  54       plane_index(0) {
  55 }
  56
  57 bool VideoResourceUpdater::PlaneResourceMatchesUniqueID(
  58     const PlaneResource& plane_resource,
  59     const media::VideoFrame* video_frame,
  60     int plane_index) {
  61   return plane_resource.frame_ptr == video_frame &&
  62          plane_resource.plane_index == plane_index &&
  63          plane_resource.timestamp == video_frame->timestamp();
  64 }
  65
  66 void VideoResourceUpdater::SetPlaneResourceUniqueId(
  67     const media::VideoFrame* video_frame,
  68     int plane_index,
  69     PlaneResource* plane_resource) {
  70   plane_resource->frame_ptr = video_frame;
  71   plane_resource->plane_index = plane_index;
  72   plane_resource->timestamp = video_frame->timestamp();
  73 }
  74
  75 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  76
  77 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  78
  79 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  80                                            ResourceProvider* resource_provider)
  81     : context_provider_(context_provider),
  82       resource_provider_(resource_provider) {
  83 }
  84
  85 VideoResourceUpdater::~VideoResourceUpdater() {
  86   for (const PlaneResource& plane_resource : all_resources_)
  87     resource_provider_->DeleteResource(plane_resource.resource_id);
  88 }
  89
  90 VideoResourceUpdater::ResourceList::iterator
  91 VideoResourceUpdater::AllocateResource(const gfx::Size& plane_size,
  92                                        ResourceFormat format,
  93                                        bool has_mailbox) {
  94   // TODO(danakj): Abstract out hw/sw resource create/delete from
  95   // ResourceProvider and stop using ResourceProvider in this class.
  96   const ResourceProvider::ResourceId resource_id =
  97       resource_provider_->CreateResource(
  98           plane_size, GL_CLAMP_TO_EDGE,
  99           ResourceProvider::TEXTURE_HINT_IMMUTABLE, format);
 100   if (resource_id == 0)
 101     return all_resources_.end();
 102
 103   gpu::Mailbox mailbox;
 104   if (has_mailbox) {
 105     DCHECK(context_provider_);
 106
 107     gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 108
 109     gl->GenMailboxCHROMIUM(mailbox.name);
 110     ResourceProvider::ScopedWriteLockGL lock(resource_provider_, resource_id);
 111     gl->ProduceTextureDirectCHROMIUM(lock.texture_id(), GL_TEXTURE_2D,
 112                                      mailbox.name);
 113   }
 114   all_resources_.push_front(
 115       PlaneResource(resource_id, plane_size, format, mailbox));
 116   return all_resources_.begin();
 117 }
 118
 119 void VideoResourceUpdater::DeleteResource(ResourceList::iterator resource_it) {
 120   DCHECK_EQ(resource_it->ref_count, 0);
 121   resource_provider_->DeleteResource(resource_it->resource_id);
 122   all_resources_.erase(resource_it);
 123 }
 124
 125 VideoFrameExternalResources VideoResourceUpdater::
 126     CreateExternalResourcesFromVideoFrame(
 127         const scoped_refptr<media::VideoFrame>& video_frame) {
 128   if (!VerifyFrame(video_frame))
 129     return VideoFrameExternalResources();
 130
 131   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
 132     return CreateForHardwarePlanes(video_frame);
 133   else
 134     return CreateForSoftwarePlanes(video_frame);
 135 }
 136
 137 bool VideoResourceUpdater::VerifyFrame(
 138     const scoped_refptr<media::VideoFrame>& video_frame) {
 139   switch (video_frame->format()) {
 140     // Acceptable inputs.
 141     case media::VideoFrame::YV12:
 142     case media::VideoFrame::I420:
 143     case media::VideoFrame::YV12A:
 144     case media::VideoFrame::YV16:
 145     case media::VideoFrame::YV12J:
 146     case media::VideoFrame::YV12HD:
 147     case media::VideoFrame::YV24:
 148     case media::VideoFrame::NATIVE_TEXTURE:
 149 #if defined(VIDEO_HOLE)
 150     case media::VideoFrame::HOLE:
 151 #endif  // defined(VIDEO_HOLE)
 152     case media::VideoFrame::ARGB:
 153       return true;
 154
 155     // Unacceptable inputs. ¯\(°_o)/¯
 156     case media::VideoFrame::UNKNOWN:
 157     case media::VideoFrame::NV12:
 158       break;
 159   }
 160   return false;
 161 }
 162
 163 // For frames that we receive in software format, determine the dimensions of
 164 // each plane in the frame.
 165 static gfx::Size SoftwarePlaneDimension(
 166     const scoped_refptr<media::VideoFrame>& input_frame,
 167     bool software_compositor,
 168     size_t plane_index) {
 169   if (!software_compositor) {
 170     return media::VideoFrame::PlaneSize(
 171         input_frame->format(), plane_index, input_frame->coded_size());
 172   }
 173   return input_frame->coded_size();
 174 }
 175
 176 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 177     const scoped_refptr<media::VideoFrame>& video_frame) {
 178   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 179   media::VideoFrame::Format input_frame_format = video_frame->format();
 180
 181 #if defined(VIDEO_HOLE)
 182   if (input_frame_format == media::VideoFrame::HOLE) {
 183     VideoFrameExternalResources external_resources;
 184     external_resources.type = VideoFrameExternalResources::HOLE;
 185     return external_resources;
 186   }
 187 #endif  // defined(VIDEO_HOLE)
 188
 189   // Only YUV software video frames are supported.
 190   if (input_frame_format != media::VideoFrame::YV12 &&
 191       input_frame_format != media::VideoFrame::I420 &&
 192       input_frame_format != media::VideoFrame::YV12A &&
 193       input_frame_format != media::VideoFrame::YV12J &&
 194       input_frame_format != media::VideoFrame::YV12HD &&
 195       input_frame_format != media::VideoFrame::YV16 &&
 196       input_frame_format != media::VideoFrame::YV24) {
 197     NOTREACHED() << input_frame_format;
 198     return VideoFrameExternalResources();
 199   }
 200
 201   bool software_compositor = context_provider_ == NULL;
 202
 203   ResourceFormat output_resource_format =
 204       resource_provider_->yuv_resource_format();
 205   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 206
 207   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 208   // conversion here. That involves an extra copy of each frame to a bitmap.
 209   // Obviously, this is suboptimal and should be addressed once ubercompositor
 210   // starts shaping up.
 211   if (software_compositor) {
 212     output_resource_format = kRGBResourceFormat;
 213     output_plane_count = 1;
 214   }
 215
 216   // Drop recycled resources that are the wrong format.
 217   for (auto it = all_resources_.begin(); it != all_resources_.end();) {
 218     if (it->ref_count == 0 && it->resource_format != output_resource_format)
 219       DeleteResource(it++);
 220     else
 221       ++it;
 222   }
 223
 224   const int max_resource_size = resource_provider_->max_texture_size();
 225   std::vector<ResourceList::iterator> plane_resources;
 226   for (size_t i = 0; i < output_plane_count; ++i) {
 227     gfx::Size output_plane_resource_size =
 228         SoftwarePlaneDimension(video_frame, software_compositor, i);
 229     if (output_plane_resource_size.IsEmpty() ||
 230         output_plane_resource_size.width() > max_resource_size ||
 231         output_plane_resource_size.height() > max_resource_size) {
 232       break;
 233     }
 234
 235     // Try recycle a previously-allocated resource.
 236     ResourceList::iterator resource_it = all_resources_.end();
 237     for (auto it = all_resources_.begin(); it != all_resources_.end(); ++it) {
 238       if (it->resource_size == output_plane_resource_size &&
 239           it->resource_format == output_resource_format) {
 240         if (PlaneResourceMatchesUniqueID(*it, video_frame.get(), i)) {
 241           // Bingo, we found a resource that already contains the data we are
 242           // planning to put in it. It's safe to reuse it even if
 243           // resource_provider_ holds some references to it, because those
 244           // references are read-only.
 245           resource_it = it;
 246           break;
 247         }
 248
 249         // This extra check is needed because resources backed by SharedMemory
 250         // are not ref-counted, unlike mailboxes. Full discussion in
 251         // codereview.chromium.org/145273021.
 252         const bool in_use =
 253             software_compositor &&
 254             resource_provider_->InUseByConsumer(it->resource_id);
 255         if (it->ref_count == 0 && !in_use) {
 256           // We found a resource with the correct size that we can overwrite.
 257           resource_it = it;
 258         }
 259       }
 260     }
 261
 262     // Check if we need to allocate a new resource.
 263     if (resource_it == all_resources_.end()) {
 264       resource_it =
 265           AllocateResource(output_plane_resource_size, output_resource_format,
 266                            !software_compositor);
 267     }
 268     if (resource_it == all_resources_.end())
 269       break;
 270
 271     ++resource_it->ref_count;
 272     plane_resources.push_back(resource_it);
 273   }
 274
 275   if (plane_resources.size() != output_plane_count) {
 276     // Allocation failed, nothing will be returned so restore reference counts.
 277     for (ResourceList::iterator resource_it : plane_resources)
 278       --resource_it->ref_count;
 279     return VideoFrameExternalResources();
 280   }
 281
 282   VideoFrameExternalResources external_resources;
 283
 284   if (software_compositor) {
 285     DCHECK_EQ(plane_resources.size(), 1u);
 286     PlaneResource& plane_resource = *plane_resources[0];
 287     DCHECK_EQ(plane_resource.resource_format, kRGBResourceFormat);
 288     DCHECK(plane_resource.mailbox.IsZero());
 289
 290     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), 0)) {
 291       // We need to transfer data from |video_frame| to the plane resource.
 292       if (!video_renderer_)
 293         video_renderer_.reset(new media::SkCanvasVideoRenderer);
 294
 295       ResourceProvider::ScopedWriteLockSoftware lock(
 296           resource_provider_, plane_resource.resource_id);
 297       SkCanvas canvas(lock.sk_bitmap());
 298       // This is software path, so canvas and video_frame are always backed
 299       // by software.
 300       video_renderer_->Copy(video_frame, &canvas, media::Context3D());
 301       SetPlaneResourceUniqueId(video_frame.get(), 0, &plane_resource);
 302     }
 303
 304     external_resources.software_resources.push_back(plane_resource.resource_id);
 305     external_resources.software_release_callback =
 306         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id);
 307     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 308     return external_resources;
 309   }
 310
 311   for (size_t i = 0; i < plane_resources.size(); ++i) {
 312     PlaneResource& plane_resource = *plane_resources[i];
 313     // Update each plane's resource id with its content.
 314     DCHECK_EQ(plane_resource.resource_format,
 315               resource_provider_->yuv_resource_format());
 316
 317     if (!PlaneResourceMatchesUniqueID(plane_resource, video_frame.get(), i)) {
 318       // We need to transfer data from |video_frame| to the plane resource.
 319       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 320
 321       // The |resource_size_pixels| is the size of the resource we want to
 322       // upload to.
 323       gfx::Size resource_size_pixels = plane_resource.resource_size;
 324       // The |video_stride_pixels| is the width of the video frame we are
 325       // uploading (including non-frame data to fill in the stride).
 326       size_t video_stride_pixels = video_frame->stride(i);
 327
 328       size_t bytes_per_pixel = BitsPerPixel(plane_resource.resource_format) / 8;
 329       // Use 4-byte row alignment (OpenGL default) for upload performance.
 330       // Assuming that GL_UNPACK_ALIGNMENT has not changed from default.
 331       size_t upload_image_stride =
 332           RoundUp<size_t>(bytes_per_pixel * resource_size_pixels.width(), 4u);
 333
 334       const uint8_t* pixels;
 335       if (upload_image_stride == video_stride_pixels * bytes_per_pixel) {
 336         pixels = video_frame->data(i);
 337       } else {
 338         // Avoid malloc for each frame/plane if possible.
 339         size_t needed_size =
 340             upload_image_stride * resource_size_pixels.height();
 341         if (upload_pixels_.size() < needed_size)
 342           upload_pixels_.resize(needed_size);
 343         for (int row = 0; row < resource_size_pixels.height(); ++row) {
 344           uint8_t* dst = &upload_pixels_[upload_image_stride * row];
 345           const uint8_t* src = video_frame->data(i) +
 346                                bytes_per_pixel * video_stride_pixels * row;
 347           memcpy(dst, src, resource_size_pixels.width() * bytes_per_pixel);
 348         }
 349         pixels = &upload_pixels_[0];
 350       }
 351
 352       resource_provider_->CopyToResource(plane_resource.resource_id, pixels,
 353                                          resource_size_pixels);
 354       SetPlaneResourceUniqueId(video_frame.get(), i, &plane_resource);
 355     }
 356
 357     external_resources.mailboxes.push_back(
 358         TextureMailbox(plane_resource.mailbox, GL_TEXTURE_2D, 0));
 359     external_resources.release_callbacks.push_back(
 360         base::Bind(&RecycleResource, AsWeakPtr(), plane_resource.resource_id));
 361   }
 362
 363   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 364   return external_resources;
 365 }
 366
 367 // static
 368 void VideoResourceUpdater::ReturnTexture(
 369     base::WeakPtr<VideoResourceUpdater> updater,
 370     const scoped_refptr<media::VideoFrame>& video_frame,
 371     uint32 sync_point,
 372     bool lost_resource,
 373     BlockingTaskRunner* main_thread_task_runner) {
 374   // TODO(dshwang) this case should be forwarded to the decoder as lost
 375   // resource.
 376   if (lost_resource || !updater.get())
 377     return;
 378   // VideoFrame::UpdateReleaseSyncPoint() creates new sync point using the same
 379   // GL context which created the given |sync_point|, so discard the
 380   // |sync_point|.
 381   SyncPointClientImpl client(updater->context_provider_->ContextGL());
 382   video_frame->UpdateReleaseSyncPoint(&client);
 383 }
 384
 385 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 386     const scoped_refptr<media::VideoFrame>& video_frame) {
 387   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 388   media::VideoFrame::Format frame_format = video_frame->format();
 389
 390   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
 391   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
 392       return VideoFrameExternalResources();
 393
 394   if (!context_provider_)
 395     return VideoFrameExternalResources();
 396
 397   const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
 398   VideoFrameExternalResources external_resources;
 399   switch (mailbox_holder->texture_target) {
 400     case GL_TEXTURE_2D:
 401       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
 402       break;
 403     case GL_TEXTURE_EXTERNAL_OES:
 404       external_resources.type =
 405           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 406       break;
 407     case GL_TEXTURE_RECTANGLE_ARB:
 408       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 409       break;
 410     default:
 411       NOTREACHED();
 412       return VideoFrameExternalResources();
 413   }
 414
 415   external_resources.mailboxes.push_back(
 416       TextureMailbox(mailbox_holder->mailbox,
 417                      mailbox_holder->texture_target,
 418                      mailbox_holder->sync_point));
 419   external_resources.mailboxes.back().set_allow_overlay(
 420       video_frame->allow_overlay());
 421   external_resources.release_callbacks.push_back(
 422       base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 423   return external_resources;
 424 }
 425
 426 // static
 427 void VideoResourceUpdater::RecycleResource(
 428     base::WeakPtr<VideoResourceUpdater> updater,
 429     ResourceProvider::ResourceId resource_id,
 430     uint32 sync_point,
 431     bool lost_resource,
 432     BlockingTaskRunner* main_thread_task_runner) {
 433   if (!updater.get()) {
 434     // Resource was already deleted.
 435     return;
 436   }
 437
 438   const ResourceList::iterator resource_it = std::find_if(
 439       updater->all_resources_.begin(), updater->all_resources_.end(),
 440       [resource_id](const PlaneResource& plane_resource) {
 441         return plane_resource.resource_id == resource_id;
 442       });
 443   if (resource_it == updater->all_resources_.end())
 444     return;
 445
 446   ContextProvider* context_provider = updater->context_provider_;
 447   if (context_provider && sync_point) {
 448     context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point);
 449   }
 450
 451   if (lost_resource) {
 452     resource_it->ref_count = 0;
 453     updater->DeleteResource(resource_it);
 454     return;
 455   }
 456
 457   --resource_it->ref_count;
 458   DCHECK_GE(resource_it->ref_count, 0);
 459 }
 460
 461 }  // namespace cc