cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include "base/bind.h"
   8 #include "base/debug/trace_event.h"
   9 #include "cc/output/gl_renderer.h"
  10 #include "cc/resources/resource_provider.h"
  11 #include "gpu/GLES2/gl2extchromium.h"
  12 #include "gpu/command_buffer/client/gles2_interface.h"
  13 #include "media/base/video_frame.h"
  14 #include "media/filters/skcanvas_video_renderer.h"
  15 #include "third_party/khronos/GLES2/gl2.h"
  16 #include "third_party/khronos/GLES2/gl2ext.h"
  17 #include "ui/gfx/geometry/size_conversions.h"
  18
  19 namespace cc {
  20
  21 namespace {
  22
  23 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
  24 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  25
  26 class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
  27  public:
  28   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
  29   ~SyncPointClientImpl() override {}
  30   uint32 InsertSyncPoint() override {
  31     return GLC(gl_, gl_->InsertSyncPointCHROMIUM());
  32   }
  33   void WaitSyncPoint(uint32 sync_point) override {
  34     GLC(gl_, gl_->WaitSyncPointCHROMIUM(sync_point));
  35   }
  36
  37  private:
  38   gpu::gles2::GLES2Interface* gl_;
  39 };
  40
  41 }  // namespace
  42
  43 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  44
  45 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  46
  47 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  48                                            ResourceProvider* resource_provider)
  49     : context_provider_(context_provider),
  50       resource_provider_(resource_provider) {
  51 }
  52
  53 VideoResourceUpdater::~VideoResourceUpdater() {
  54   while (!all_resources_.empty()) {
  55     resource_provider_->DeleteResource(all_resources_.back());
  56     all_resources_.pop_back();
  57   }
  58 }
  59
  60 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
  61   resource_provider_->DeleteResource(resource_id);
  62   all_resources_.erase(std::remove(all_resources_.begin(),
  63                                    all_resources_.end(),
  64                                    resource_id));
  65 }
  66
  67 VideoFrameExternalResources VideoResourceUpdater::
  68     CreateExternalResourcesFromVideoFrame(
  69         const scoped_refptr<media::VideoFrame>& video_frame) {
  70   if (!VerifyFrame(video_frame))
  71     return VideoFrameExternalResources();
  72
  73   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
  74     return CreateForHardwarePlanes(video_frame);
  75   else
  76     return CreateForSoftwarePlanes(video_frame);
  77 }
  78
  79 bool VideoResourceUpdater::VerifyFrame(
  80     const scoped_refptr<media::VideoFrame>& video_frame) {
  81   switch (video_frame->format()) {
  82     // Acceptable inputs.
  83     case media::VideoFrame::YV12:
  84     case media::VideoFrame::I420:
  85     case media::VideoFrame::YV12A:
  86     case media::VideoFrame::YV16:
  87     case media::VideoFrame::YV12J:
  88     case media::VideoFrame::YV24:
  89     case media::VideoFrame::NATIVE_TEXTURE:
  90 #if defined(VIDEO_HOLE)
  91     case media::VideoFrame::HOLE:
  92 #endif  // defined(VIDEO_HOLE)
  93       return true;
  94
  95     // Unacceptable inputs. ¯\(°_o)/¯
  96     case media::VideoFrame::UNKNOWN:
  97     case media::VideoFrame::NV12:
  98       break;
  99   }
 100   return false;
 101 }
 102
 103 // For frames that we receive in software format, determine the dimensions of
 104 // each plane in the frame.
 105 static gfx::Size SoftwarePlaneDimension(
 106     const scoped_refptr<media::VideoFrame>& input_frame,
 107     ResourceFormat output_resource_format,
 108     size_t plane_index) {
 109   if (output_resource_format == kYUVResourceFormat) {
 110     return media::VideoFrame::PlaneSize(
 111         input_frame->format(), plane_index, input_frame->coded_size());
 112   }
 113
 114   DCHECK_EQ(output_resource_format, kRGBResourceFormat);
 115   return input_frame->coded_size();
 116 }
 117
 118 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 119     const scoped_refptr<media::VideoFrame>& video_frame) {
 120   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
 121   media::VideoFrame::Format input_frame_format = video_frame->format();
 122
 123 #if defined(VIDEO_HOLE)
 124   if (input_frame_format == media::VideoFrame::HOLE) {
 125     VideoFrameExternalResources external_resources;
 126     external_resources.type = VideoFrameExternalResources::HOLE;
 127     return external_resources;
 128   }
 129 #endif  // defined(VIDEO_HOLE)
 130
 131   // Only YUV software video frames are supported.
 132   DCHECK(input_frame_format == media::VideoFrame::YV12 ||
 133          input_frame_format == media::VideoFrame::I420 ||
 134          input_frame_format == media::VideoFrame::YV12A ||
 135          input_frame_format == media::VideoFrame::YV12J ||
 136          input_frame_format == media::VideoFrame::YV16 ||
 137          input_frame_format == media::VideoFrame::YV24);
 138   if (input_frame_format != media::VideoFrame::YV12 &&
 139       input_frame_format != media::VideoFrame::I420 &&
 140       input_frame_format != media::VideoFrame::YV12A &&
 141       input_frame_format != media::VideoFrame::YV12J &&
 142       input_frame_format != media::VideoFrame::YV16 &&
 143       input_frame_format != media::VideoFrame::YV24)
 144     return VideoFrameExternalResources();
 145
 146   bool software_compositor = context_provider_ == NULL;
 147
 148   ResourceFormat output_resource_format = kYUVResourceFormat;
 149   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 150
 151   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 152   // conversion here. That involves an extra copy of each frame to a bitmap.
 153   // Obviously, this is suboptimal and should be addressed once ubercompositor
 154   // starts shaping up.
 155   if (software_compositor) {
 156     output_resource_format = kRGBResourceFormat;
 157     output_plane_count = 1;
 158   }
 159
 160   int max_resource_size = resource_provider_->max_texture_size();
 161   std::vector<PlaneResource> plane_resources;
 162   bool allocation_success = true;
 163
 164   for (size_t i = 0; i < output_plane_count; ++i) {
 165     gfx::Size output_plane_resource_size =
 166         SoftwarePlaneDimension(video_frame, output_resource_format, i);
 167     if (output_plane_resource_size.IsEmpty() ||
 168         output_plane_resource_size.width() > max_resource_size ||
 169         output_plane_resource_size.height() > max_resource_size) {
 170       allocation_success = false;
 171       break;
 172     }
 173
 174     ResourceProvider::ResourceId resource_id = 0;
 175     gpu::Mailbox mailbox;
 176
 177     // Try recycle a previously-allocated resource.
 178     for (size_t i = 0; i < recycled_resources_.size(); ++i) {
 179       bool resource_matches =
 180           recycled_resources_[i].resource_format == output_resource_format &&
 181           recycled_resources_[i].resource_size == output_plane_resource_size;
 182       bool not_in_use =
 183           !software_compositor || !resource_provider_->InUseByConsumer(
 184                                        recycled_resources_[i].resource_id);
 185       if (resource_matches && not_in_use) {
 186         resource_id = recycled_resources_[i].resource_id;
 187         mailbox = recycled_resources_[i].mailbox;
 188         recycled_resources_.erase(recycled_resources_.begin() + i);
 189         break;
 190       }
 191     }
 192
 193     if (resource_id == 0) {
 194       // TODO(danakj): Abstract out hw/sw resource create/delete from
 195       // ResourceProvider and stop using ResourceProvider in this class.
 196       resource_id = resource_provider_->CreateResource(
 197           output_plane_resource_size,
 198           GL_CLAMP_TO_EDGE,
 199           ResourceProvider::TextureHintImmutable,
 200           output_resource_format);
 201
 202       DCHECK(mailbox.IsZero());
 203
 204       if (!software_compositor) {
 205         DCHECK(context_provider_);
 206
 207         gpu::gles2::GLES2Interface* gl = context_provider_->ContextGL();
 208
 209         GLC(gl, gl->GenMailboxCHROMIUM(mailbox.name));
 210         ResourceProvider::ScopedWriteLockGL lock(resource_provider_,
 211                                                  resource_id);
 212         GLC(gl,
 213             gl->ProduceTextureDirectCHROMIUM(
 214                 lock.texture_id(), GL_TEXTURE_2D, mailbox.name));
 215       }
 216
 217       if (resource_id)
 218         all_resources_.push_back(resource_id);
 219     }
 220
 221     if (resource_id == 0) {
 222       allocation_success = false;
 223       break;
 224     }
 225
 226     DCHECK(software_compositor || !mailbox.IsZero());
 227     plane_resources.push_back(PlaneResource(resource_id,
 228                                             output_plane_resource_size,
 229                                             output_resource_format,
 230                                             mailbox));
 231   }
 232
 233   if (!allocation_success) {
 234     for (size_t i = 0; i < plane_resources.size(); ++i)
 235       DeleteResource(plane_resources[i].resource_id);
 236     return VideoFrameExternalResources();
 237   }
 238
 239   VideoFrameExternalResources external_resources;
 240
 241   if (software_compositor) {
 242     DCHECK_EQ(plane_resources.size(), 1u);
 243     DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
 244     DCHECK(plane_resources[0].mailbox.IsZero());
 245
 246     if (!video_renderer_)
 247       video_renderer_.reset(new media::SkCanvasVideoRenderer);
 248
 249     {
 250       ResourceProvider::ScopedWriteLockSoftware lock(
 251           resource_provider_, plane_resources[0].resource_id);
 252       video_renderer_->Copy(video_frame, lock.sk_canvas());
 253     }
 254
 255     RecycleResourceData recycle_data = {
 256       plane_resources[0].resource_id,
 257       plane_resources[0].resource_size,
 258       plane_resources[0].resource_format,
 259       gpu::Mailbox()
 260     };
 261     external_resources.software_resources.push_back(
 262         plane_resources[0].resource_id);
 263     external_resources.software_release_callback =
 264         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
 265     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 266
 267     return external_resources;
 268   }
 269
 270   for (size_t i = 0; i < plane_resources.size(); ++i) {
 271     // Update each plane's resource id with its content.
 272     DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
 273
 274     const uint8_t* input_plane_pixels = video_frame->data(i);
 275
 276     gfx::Rect image_rect(0,
 277                          0,
 278                          video_frame->stride(i),
 279                          plane_resources[i].resource_size.height());
 280     gfx::Rect source_rect(plane_resources[i].resource_size);
 281     resource_provider_->SetPixels(plane_resources[i].resource_id,
 282                                   input_plane_pixels,
 283                                   image_rect,
 284                                   source_rect,
 285                                   gfx::Vector2d());
 286
 287     RecycleResourceData recycle_data = {
 288       plane_resources[i].resource_id,
 289       plane_resources[i].resource_size,
 290       plane_resources[i].resource_format,
 291       plane_resources[i].mailbox
 292     };
 293
 294     external_resources.mailboxes.push_back(
 295         TextureMailbox(plane_resources[i].mailbox, GL_TEXTURE_2D, 0));
 296     external_resources.release_callbacks.push_back(
 297         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
 298   }
 299
 300   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 301   return external_resources;
 302 }
 303
 304 // static
 305 void VideoResourceUpdater::ReturnTexture(
 306     base::WeakPtr<VideoResourceUpdater> updater,
 307     const scoped_refptr<media::VideoFrame>& video_frame,
 308     uint32 sync_point,
 309     bool lost_resource,
 310     BlockingTaskRunner* main_thread_task_runner) {
 311   // TODO(dshwang) this case should be forwarded to the decoder as lost
 312   // resource.
 313   if (lost_resource || !updater.get())
 314     return;
 315   // VideoFrame::UpdateReleaseSyncPoint() creates new sync point using the same
 316   // GL context which created the given |sync_point|, so discard the
 317   // |sync_point|.
 318   SyncPointClientImpl client(updater->context_provider_->ContextGL());
 319   video_frame->UpdateReleaseSyncPoint(&client);
 320 }
 321
 322 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 323     const scoped_refptr<media::VideoFrame>& video_frame) {
 324   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForHardwarePlanes");
 325   media::VideoFrame::Format frame_format = video_frame->format();
 326
 327   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
 328   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
 329       return VideoFrameExternalResources();
 330
 331   if (!context_provider_)
 332     return VideoFrameExternalResources();
 333
 334   const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
 335   VideoFrameExternalResources external_resources;
 336   switch (mailbox_holder->texture_target) {
 337     case GL_TEXTURE_2D:
 338       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
 339       break;
 340     case GL_TEXTURE_EXTERNAL_OES:
 341       external_resources.type =
 342           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 343       break;
 344     case GL_TEXTURE_RECTANGLE_ARB:
 345       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 346       break;
 347     default:
 348       NOTREACHED();
 349       return VideoFrameExternalResources();
 350   }
 351
 352   external_resources.mailboxes.push_back(
 353       TextureMailbox(mailbox_holder->mailbox,
 354                      mailbox_holder->texture_target,
 355                      mailbox_holder->sync_point));
 356   external_resources.release_callbacks.push_back(
 357       base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
 358   return external_resources;
 359 }
 360
 361 // static
 362 void VideoResourceUpdater::RecycleResource(
 363     base::WeakPtr<VideoResourceUpdater> updater,
 364     RecycleResourceData data,
 365     uint32 sync_point,
 366     bool lost_resource,
 367     BlockingTaskRunner* main_thread_task_runner) {
 368   if (!updater.get()) {
 369     // Resource was already deleted.
 370     return;
 371   }
 372
 373   ContextProvider* context_provider = updater->context_provider_;
 374   if (context_provider && sync_point) {
 375     GLC(context_provider->ContextGL(),
 376         context_provider->ContextGL()->WaitSyncPointCHROMIUM(sync_point));
 377   }
 378
 379   if (lost_resource) {
 380     updater->DeleteResource(data.resource_id);
 381     return;
 382   }
 383
 384   // Drop recycled resources that are the wrong format.
 385   while (!updater->recycled_resources_.empty() &&
 386          updater->recycled_resources_.back().resource_format !=
 387          data.resource_format) {
 388     updater->DeleteResource(updater->recycled_resources_.back().resource_id);
 389     updater->recycled_resources_.pop_back();
 390   }
 391
 392   PlaneResource recycled_resource(data.resource_id,
 393                                   data.resource_size,
 394                                   data.resource_format,
 395                                   data.mailbox);
 396   updater->recycled_resources_.push_back(recycled_resource);
 397 }
 398
 399 }  // namespace cc