cc/resources/video_resource_updater.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "cc/resources/video_resource_updater.h"
   6
   7 #include "base/bind.h"
   8 #include "cc/output/gl_renderer.h"
   9 #include "cc/resources/resource_provider.h"
  10 #include "gpu/GLES2/gl2extchromium.h"
  11 #include "media/base/video_frame.h"
  12 #include "media/filters/skcanvas_video_renderer.h"
  13 #include "third_party/WebKit/public/platform/WebGraphicsContext3D.h"
  14 #include "third_party/khronos/GLES2/gl2.h"
  15 #include "third_party/khronos/GLES2/gl2ext.h"
  16 #include "ui/gfx/size_conversions.h"
  17
  18 namespace cc {
  19
  20 const ResourceFormat kYUVResourceFormat = LUMINANCE_8;
  21 const ResourceFormat kRGBResourceFormat = RGBA_8888;
  22
  23 VideoFrameExternalResources::VideoFrameExternalResources() : type(NONE) {}
  24
  25 VideoFrameExternalResources::~VideoFrameExternalResources() {}
  26
  27 VideoResourceUpdater::VideoResourceUpdater(ContextProvider* context_provider,
  28                                            ResourceProvider* resource_provider)
  29     : context_provider_(context_provider),
  30       resource_provider_(resource_provider) {
  31 }
  32
  33 VideoResourceUpdater::~VideoResourceUpdater() {
  34   while (!all_resources_.empty()) {
  35     resource_provider_->DeleteResource(all_resources_.back());
  36     all_resources_.pop_back();
  37   }
  38 }
  39
  40 void VideoResourceUpdater::DeleteResource(unsigned resource_id) {
  41   resource_provider_->DeleteResource(resource_id);
  42   all_resources_.erase(std::remove(all_resources_.begin(),
  43                                    all_resources_.end(),
  44                                    resource_id));
  45 }
  46
  47 VideoFrameExternalResources VideoResourceUpdater::
  48     CreateExternalResourcesFromVideoFrame(
  49         const scoped_refptr<media::VideoFrame>& video_frame) {
  50   if (!VerifyFrame(video_frame))
  51     return VideoFrameExternalResources();
  52
  53   if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)
  54     return CreateForHardwarePlanes(video_frame);
  55   else
  56     return CreateForSoftwarePlanes(video_frame);
  57 }
  58
  59 bool VideoResourceUpdater::VerifyFrame(
  60     const scoped_refptr<media::VideoFrame>& video_frame) {
  61   // If these fail, we'll have to add logic that handles offset bitmap/texture
  62   // UVs. For now, just expect (0, 0) offset, since all our decoders so far
  63   // don't offset.
  64   DCHECK_EQ(video_frame->visible_rect().x(), 0);
  65   DCHECK_EQ(video_frame->visible_rect().y(), 0);
  66
  67   switch (video_frame->format()) {
  68     // Acceptable inputs.
  69     case media::VideoFrame::YV12:
  70     case media::VideoFrame::YV12A:
  71     case media::VideoFrame::YV16:
  72     case media::VideoFrame::NATIVE_TEXTURE:
  73 #if defined(GOOGLE_TV)
  74     case media::VideoFrame::HOLE:
  75 #endif
  76       return true;
  77
  78     // Unacceptable inputs. ¯\(°_o)/¯
  79     case media::VideoFrame::INVALID:
  80     case media::VideoFrame::RGB32:
  81     case media::VideoFrame::EMPTY:
  82     case media::VideoFrame::I420:
  83       break;
  84   }
  85   return false;
  86 }
  87
  88 // For frames that we receive in software format, determine the dimensions of
  89 // each plane in the frame.
  90 static gfx::Size SoftwarePlaneDimension(
  91     media::VideoFrame::Format input_frame_format,
  92     gfx::Size coded_size,
  93     ResourceFormat output_resource_format,
  94     int plane_index) {
  95   if (output_resource_format == kYUVResourceFormat) {
  96     if (plane_index == media::VideoFrame::kYPlane ||
  97         plane_index == media::VideoFrame::kAPlane)
  98       return coded_size;
  99
 100     switch (input_frame_format) {
 101       case media::VideoFrame::YV12:
 102       case media::VideoFrame::YV12A:
 103         return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 0.5f));
 104       case media::VideoFrame::YV16:
 105         return gfx::ToFlooredSize(gfx::ScaleSize(coded_size, 0.5f, 1.f));
 106
 107       case media::VideoFrame::INVALID:
 108       case media::VideoFrame::RGB32:
 109       case media::VideoFrame::EMPTY:
 110       case media::VideoFrame::I420:
 111       case media::VideoFrame::NATIVE_TEXTURE:
 112 #if defined(GOOGLE_TV)
 113       case media::VideoFrame::HOLE:
 114 #endif
 115         NOTREACHED();
 116     }
 117   }
 118
 119   DCHECK_EQ(output_resource_format, kRGBResourceFormat);
 120   return coded_size;
 121 }
 122
 123 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
 124     const scoped_refptr<media::VideoFrame>& video_frame) {
 125   media::VideoFrame::Format input_frame_format = video_frame->format();
 126
 127 #if defined(GOOGLE_TV)
 128   if (input_frame_format == media::VideoFrame::HOLE) {
 129     VideoFrameExternalResources external_resources;
 130     external_resources.type = VideoFrameExternalResources::HOLE;
 131     return external_resources;
 132   }
 133 #endif
 134
 135   // Only YUV software video frames are supported.
 136   DCHECK(input_frame_format == media::VideoFrame::YV12 ||
 137          input_frame_format == media::VideoFrame::YV12A ||
 138          input_frame_format == media::VideoFrame::YV16);
 139   if (input_frame_format != media::VideoFrame::YV12 &&
 140       input_frame_format != media::VideoFrame::YV12A &&
 141       input_frame_format != media::VideoFrame::YV16)
 142     return VideoFrameExternalResources();
 143
 144   bool software_compositor = context_provider_ == NULL;
 145
 146   ResourceFormat output_resource_format = kYUVResourceFormat;
 147   size_t output_plane_count =
 148       (input_frame_format == media::VideoFrame::YV12A) ? 4 : 3;
 149
 150   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
 151   // conversion here. That involves an extra copy of each frame to a bitmap.
 152   // Obviously, this is suboptimal and should be addressed once ubercompositor
 153   // starts shaping up.
 154   if (software_compositor) {
 155     output_resource_format = kRGBResourceFormat;
 156     output_plane_count = 1;
 157   }
 158
 159   int max_resource_size = resource_provider_->max_texture_size();
 160   gfx::Size coded_frame_size = video_frame->coded_size();
 161
 162   std::vector<PlaneResource> plane_resources;
 163   bool allocation_success = true;
 164
 165   for (size_t i = 0; i < output_plane_count; ++i) {
 166     gfx::Size output_plane_resource_size =
 167         SoftwarePlaneDimension(input_frame_format,
 168                                coded_frame_size,
 169                                output_resource_format,
 170                                i);
 171     if (output_plane_resource_size.IsEmpty() ||
 172         output_plane_resource_size.width() > max_resource_size ||
 173         output_plane_resource_size.height() > max_resource_size) {
 174       allocation_success = false;
 175       break;
 176     }
 177
 178     ResourceProvider::ResourceId resource_id = 0;
 179     gpu::Mailbox mailbox;
 180
 181     // Try recycle a previously-allocated resource.
 182     for (size_t i = 0; i < recycled_resources_.size(); ++i) {
 183       if (recycled_resources_[i].resource_format == output_resource_format &&
 184           recycled_resources_[i].resource_size == output_plane_resource_size) {
 185         resource_id = recycled_resources_[i].resource_id;
 186         mailbox = recycled_resources_[i].mailbox;
 187         recycled_resources_.erase(recycled_resources_.begin() + i);
 188         break;
 189       }
 190     }
 191
 192     if (resource_id == 0) {
 193       // TODO(danakj): Abstract out hw/sw resource create/delete from
 194       // ResourceProvider and stop using ResourceProvider in this class.
 195       resource_id =
 196           resource_provider_->CreateResource(output_plane_resource_size,
 197                                              GL_CLAMP_TO_EDGE,
 198                                              ResourceProvider::TextureUsageAny,
 199                                              output_resource_format);
 200
 201       DCHECK(mailbox.IsZero());
 202
 203       if (!software_compositor) {
 204         DCHECK(context_provider_);
 205
 206         WebKit::WebGraphicsContext3D* context =
 207             context_provider_->Context3d();
 208
 209         GLC(context, context->genMailboxCHROMIUM(mailbox.name));
 210         if (mailbox.IsZero()) {
 211           resource_provider_->DeleteResource(resource_id);
 212           resource_id = 0;
 213         } else {
 214           ResourceProvider::ScopedWriteLockGL lock(
 215               resource_provider_, resource_id);
 216           GLC(context, context->bindTexture(GL_TEXTURE_2D, lock.texture_id()));
 217           GLC(context, context->produceTextureCHROMIUM(GL_TEXTURE_2D,
 218                                                        mailbox.name));
 219           GLC(context, context->bindTexture(GL_TEXTURE_2D, 0));
 220         }
 221       }
 222
 223       if (resource_id)
 224         all_resources_.push_back(resource_id);
 225     }
 226
 227     if (resource_id == 0) {
 228       allocation_success = false;
 229       break;
 230     }
 231
 232     DCHECK(software_compositor || !mailbox.IsZero());
 233     plane_resources.push_back(PlaneResource(resource_id,
 234                                             output_plane_resource_size,
 235                                             output_resource_format,
 236                                             mailbox));
 237   }
 238
 239   if (!allocation_success) {
 240     for (size_t i = 0; i < plane_resources.size(); ++i)
 241       DeleteResource(plane_resources[i].resource_id);
 242     return VideoFrameExternalResources();
 243   }
 244
 245   VideoFrameExternalResources external_resources;
 246
 247   if (software_compositor) {
 248     DCHECK_EQ(plane_resources.size(), 1u);
 249     DCHECK_EQ(plane_resources[0].resource_format, kRGBResourceFormat);
 250     DCHECK(plane_resources[0].mailbox.IsZero());
 251
 252     if (!video_renderer_)
 253       video_renderer_.reset(new media::SkCanvasVideoRenderer);
 254
 255     {
 256       ResourceProvider::ScopedWriteLockSoftware lock(
 257           resource_provider_, plane_resources[0].resource_id);
 258       video_renderer_->Paint(video_frame.get(),
 259                              lock.sk_canvas(),
 260                              video_frame->visible_rect(),
 261                              0xff);
 262     }
 263
 264     // In software mode, the resource provider won't be lost. Soon this callback
 265     // will be called directly from the resource provider, same as 3d
 266     // compositing mode, so this raw unretained resource_provider will always
 267     // be valid when the callback is fired.
 268     RecycleResourceData recycle_data = {
 269       plane_resources[0].resource_id,
 270       plane_resources[0].resource_size,
 271       plane_resources[0].resource_format,
 272       gpu::Mailbox()
 273     };
 274
 275     external_resources.software_resources.push_back(
 276         plane_resources[0].resource_id);
 277     external_resources.software_release_callback =
 278         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data);
 279
 280
 281     external_resources.type = VideoFrameExternalResources::SOFTWARE_RESOURCE;
 282     return external_resources;
 283   }
 284
 285   for (size_t i = 0; i < plane_resources.size(); ++i) {
 286     // Update each plane's resource id with its content.
 287     DCHECK_EQ(plane_resources[i].resource_format, kYUVResourceFormat);
 288
 289     const uint8_t* input_plane_pixels = video_frame->data(i);
 290
 291     gfx::Rect image_rect(0,
 292                          0,
 293                          video_frame->stride(i),
 294                          plane_resources[i].resource_size.height());
 295     gfx::Rect source_rect(plane_resources[i].resource_size);
 296     resource_provider_->SetPixels(plane_resources[i].resource_id,
 297                                   input_plane_pixels,
 298                                   image_rect,
 299                                   source_rect,
 300                                   gfx::Vector2d());
 301
 302     RecycleResourceData recycle_data = {
 303       plane_resources[i].resource_id,
 304       plane_resources[i].resource_size,
 305       plane_resources[i].resource_format,
 306       plane_resources[i].mailbox
 307     };
 308
 309     external_resources.mailboxes.push_back(
 310         TextureMailbox(plane_resources[i].mailbox));
 311     external_resources.release_callbacks.push_back(
 312         base::Bind(&RecycleResource, AsWeakPtr(), recycle_data));
 313   }
 314
 315   external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
 316   return external_resources;
 317 }
 318
 319 static void ReturnTexture(
 320     scoped_refptr<media::VideoFrame::MailboxHolder> mailbox_holder,
 321     unsigned sync_point,
 322     bool lost_resource) {
 323   mailbox_holder->Return(sync_point);
 324 }
 325
 326 VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
 327     const scoped_refptr<media::VideoFrame>& video_frame) {
 328   media::VideoFrame::Format frame_format = video_frame->format();
 329
 330   DCHECK_EQ(frame_format, media::VideoFrame::NATIVE_TEXTURE);
 331   if (frame_format != media::VideoFrame::NATIVE_TEXTURE)
 332       return VideoFrameExternalResources();
 333
 334   if (!context_provider_)
 335     return VideoFrameExternalResources();
 336
 337   VideoFrameExternalResources external_resources;
 338   switch (video_frame->texture_target()) {
 339     case GL_TEXTURE_2D:
 340       external_resources.type = VideoFrameExternalResources::RGB_RESOURCE;
 341       break;
 342     case GL_TEXTURE_EXTERNAL_OES:
 343       external_resources.type =
 344           VideoFrameExternalResources::STREAM_TEXTURE_RESOURCE;
 345       break;
 346     case GL_TEXTURE_RECTANGLE_ARB:
 347       external_resources.type = VideoFrameExternalResources::IO_SURFACE;
 348       break;
 349     default:
 350       NOTREACHED();
 351       return VideoFrameExternalResources();
 352   }
 353
 354   scoped_refptr<media::VideoFrame::MailboxHolder> mailbox_holder =
 355       video_frame->texture_mailbox();
 356
 357   external_resources.mailboxes.push_back(
 358       TextureMailbox(mailbox_holder->mailbox(),
 359                      video_frame->texture_target(),
 360                      mailbox_holder->sync_point()));
 361   external_resources.release_callbacks.push_back(
 362       base::Bind(&ReturnTexture, mailbox_holder));
 363   return external_resources;
 364 }
 365
 366 // static
 367 void VideoResourceUpdater::RecycleResource(
 368     base::WeakPtr<VideoResourceUpdater> updater,
 369     RecycleResourceData data,
 370     unsigned sync_point,
 371     bool lost_resource) {
 372   if (!updater.get()) {
 373     // Resource was already deleted.
 374     return;
 375   }
 376
 377   ContextProvider* context_provider = updater->context_provider_;
 378   if (context_provider && sync_point) {
 379     GLC(context_provider->Context3d(),
 380         context_provider->Context3d()->waitSyncPoint(sync_point));
 381   }
 382
 383   if (lost_resource) {
 384     updater->DeleteResource(data.resource_id);
 385     return;
 386   }
 387
 388   // Drop recycled resources that are the wrong format.
 389   while (!updater->recycled_resources_.empty() &&
 390          updater->recycled_resources_.back().resource_format !=
 391          data.resource_format) {
 392     updater->DeleteResource(updater->recycled_resources_.back().resource_id);
 393     updater->recycled_resources_.pop_back();
 394   }
 395
 396   PlaneResource recycled_resource(data.resource_id,
 397                                   data.resource_size,
 398                                   data.resource_format,
 399                                   data.mailbox);
 400   updater->recycled_resources_.push_back(recycled_resource);
 401 }
 402
 403 }  // namespace cc